/*
 * hugepage-mmap:
 *
 * Example of using huge page memory in a user application using the mmap
 * system call.  Before running this application, make sure that the
 * administrator has mounted the hugetlbfs filesystem (on some directory
 * like /mnt) using the command mount -t hugetlbfs nodev /mnt. In this
 * example, the app is requesting memory of size 256MB that is backed by
 * huge pages.
 *
 * For the ia64 architecture, the Linux kernel reserves Region number 4 for
 * huge pages.  That means that if one requires a fixed address, a huge page
 * aligned address starting with 0x800000... will be required.  If a fixed
 * address is not required, the kernel will select an address in the proper
 * range.
 * Other architectures, such as ppc64, i386 or x86_64 are not so constrained.
 */

#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <sys/mman.h>
#define __USE_GNU
#include <fcntl.h>
#include <errno.h>
#include <sys/types.h>
#include <time.h>
#include <numa.h>
#include <numaif.h>

#define USAGE "USAGE: %s num_hpages hugepagefile_name"
#define H_PAGESIZE (2 * 1024 * 1024)
#define B_PAGESIZE (4096)

#define ITERATIONS 100000

#define PROTECTION (PROT_READ | PROT_WRITE)
#define ADDR (void *)(0x0UL)
#define FLAGS (MAP_SHARED)

int main(int argc, char ** argv)
{
	char *f_name;
	char *sep;
	char ch;
	int fd;
	long i;
	long long hpages, bpages;
	void *addr;
	char foo;
	long count = 0;
	void **pages;
	int *nodes;
	int *status;
	int flags;
	long m_ret;
	/*
	 * HARD CODED FOR TWO NODES: 0 and 1
	 */
	unsigned long node0_mask = 01L << 0;
	unsigned long node1_mask = 01L << 1;

	if (argc != 3) {
		printf(USAGE, argv[0]);
		exit (1);
	}

	hpages = strtol(argv[1], &sep, 0);
	if (errno || hpages < 0) {
		printf("Invalid number hpages (%s)\n", argv[1]);
		printf(USAGE, argv[0]);
		exit (1);
	}
	bpages = hpages * (H_PAGESIZE / B_PAGESIZE);

	f_name = argv[2];
	fd = open(f_name, O_CREAT | O_RDWR, 0755);
	if (fd < 0) {
		printf("Open of %s failed", argv[2]);
		exit(1);
	}

	addr = mmap(ADDR, hpages * H_PAGESIZE, PROTECTION, FLAGS, fd, 0);
	if (addr == MAP_FAILED) {
		perror("mmap");
		exit (1);
	}
	printf("%ld huge pages mapped at 0x%lx\n", hpages,
		( unsigned long)addr);
	printf("Faulting in all pages\n");
	for (i=0; i < hpages; i++)
		foo = *((char *)(addr + (i * H_PAGESIZE)));

	pages = malloc(bpages * sizeof(void *));
	nodes = malloc(bpages * sizeof(int));
	status = malloc(bpages * sizeof(int));
	if (!pages || !nodes || !status) {
		printf("error allocating memory for arrays\n");
		exit (1);
	}

while (1) {
	printf("Hit any key to move hugetlb pages to node 1\n");
	read(STDIN_FILENO, &ch, 1);

	for (i=0; i < hpages; i++) {
		pages[i] = addr + (i * H_PAGESIZE);
		// pages[i] = addr + (i * H_PAGESIZE) + B_PAGESIZE;
		nodes[i] = 1;
		status[i] = -1;
		flags = MPOL_MF_MOVE_ALL;
	}
	m_ret = numa_move_pages(0, hpages, pages, nodes, status, flags);
	if (m_ret) {
		perror("move_pages");
		if (m_ret > 0)
			printf("%ld pages not migrated\n", m_ret);
	} else {
		printf("Success!\n");
	}
	for (i=0; i < hpages; i++) {
		printf("\tstatus[%d] = %d\n", i, status[i]);
		status[i] = -1;
	}

	printf("Hit any key to move hugetlb pages to node 0\n");
	read(STDIN_FILENO, &ch, 1);
	for (i=0; i < hpages; i++) {
		pages[i] = addr + (i * H_PAGESIZE);
		// pages[i] = addr + (i * H_PAGESIZE) + B_PAGESIZE;
		nodes[i] = 0;
		status[i] = -1;
		flags = MPOL_MF_MOVE_ALL;
	}
	m_ret = numa_move_pages(0, hpages, pages, nodes, status, flags);
	if (m_ret) {
		perror("move_pages");
		if (m_ret > 0)
			printf("%ld pages not migrated\n", m_ret);
	} else {
		printf("Success!\n");
	}
	for (i=0; i < hpages; i++) {
		printf("\tstatus[%d] = %d\n", i, status[i]);
		status[i] = -1;
	}
}

	munmap(addr, hpages * H_PAGESIZE);
	close(fd);

	return 0;
}