Conference turris::decc

   What this sounds like is a pagefile leak -- this is most often caused
   by an application that is using and not freeing memory, via continual
   virtual address space increases.  This is most often due to a failure
   to free dynamically allocated memory, but there are certainly other
   causes.

   I'd recommend rolling one's own memory management wrappers around the
   malloc and free, or -- better -- calling the lib$vm routines directly.
   (With the direct calls, one can use the vm statistics routines, and one
   can better tailor the memory management processing and performance via
   the use of various lookaside lists and memory zones.  Further, one can
   code the application to use the zones to flush "all" memory related to
   a particular (completed) task.  And one can use these wrappers to keep
   track of vm corrupters, and unmatched vm allocation-free pairs.)

   You may end up chasing a VM leak in one the packages being called, or
   programming errors in calling OpenVMS system services, etc.

    Could anyone provide a sample source for keep tracking of virtual memory
    statistics, unmatched vm allocation-free pair, etc?
    
    Thanks.
    	JH Lee

:    Could anyone provide a sample source for keep tracking of virtual memory
:    statistics, unmatched vm allocation-free pair, etc?

    See the LIB$SHOW_VM_* service documentation, and see the memory allocation
    and deallocation chapter in the OpenVMS Programming Concepts manual.

    I wrote a real simple program using malloc,dealloc,lib$show_vm and found 
    that malloc calls lib$get_vm and free calls lib$free_vm.
    I watched the process using $sh proc/cont.
    Working Set and Virtual Page grew and No of lib$get_vm_page calls increased
    as malloc() succeeded, but Working Set and Virtual Page never goes down
    again even when free() succeded and lib$show_vm showed allocated memory size
    was also decreasing.
    
    Is it normal and why?
    If it's not the main cause of pagefile problem, what are the typical
    causes of it?
    Sorry if this has to be posted on other notes.
    
    Thank You
            Lee
                                      

/*
** In each Source file of the watched program, do the following:
** extern void * malloc1(int)
** extern void free(void *)
** #define malloc malloc1
** #define free free1
**
** In a separate source file:
** void * malloc1(int size){
** char *buf;
**
** buf = malloc(size);
** printf ("%x = malloc (%d)\n",buf,size);
** return (buf);
** }
**
** void free1(void *buf){
** printf ("free(%x)\n",buf);
** free(buf);
** }
**
** When the application is ran, do $ define sys$output toto.log
*/

#define TABLEN	100000

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

struct malloc_free {
	int iPointer;
	int iSize;
	short num_alloc;
	short num_free;
};

int MAX(a, b)
int a, b;
{
if (a > b)
   return a;
else
   return b;
}

int MIN(a, b)
int a, b;
{
if (a < b)
   return a;
else
   return b;
}

int sort_address (struct malloc_free *t1,struct malloc_free *t2)
{
if (t1->iPointer == 0) return(-1);
return (t1->iPointer - t2->iPointer);
}

int main(argc, argv)
int argc;
char **argv;
{
FILE *fp;
char ligne[100];
char ss[10];
int iPointer, iSize;
int iMinPointer,iMaxPointer;
int i,j, iLength;
struct malloc_free *tPointer;
unsigned int TotalFree,TotalSize;
char equal,open_bracket,close_bracket;
int sort_address();

fp = fopen("toto.log", "r");
iLength = iMaxPointer = 0;
tPointer = calloc(TABLEN,sizeof(struct malloc_free));
while ( fgets(&ligne[0],100,fp) != NULL) {
     if (strstr(ligne, "= malloc (") != NULL ) {
        sscanf (ligne,"%x %c %6s %c%x%c\n",&iPointer,&equal,ss,&open_bracket,&iSize,&close_bracket);
	i = 0;
	while ((tPointer[i].iPointer != iPointer) &&
		(tPointer[i].iPointer != 0)) {
	  i++;
	  if (i > TABLEN) {
	    printf("TABLEN too small.\n");
	    exit(0);
	  }
	}
        if (i == 0) iMinPointer = iPointer;
	else iMinPointer = MIN(iMinPointer,iPointer);
	iMaxPointer = MAX(iMaxPointer,iPointer);
	tPointer[i].iPointer= iPointer;
	tPointer[i].iSize=iSize;
	tPointer[i].num_alloc++;
	iLength = MAX(i, iLength);
     } /* end if (strstr(ligne,"= malloc (") != NULL) */

     if (strstr(ligne, "free (")  != NULL) {
        sscanf (ligne,"%4s %c%x%c\n",ss,&open_bracket,&iPointer,&close_bracket);
	i = 0;
	while ((tPointer[i].iPointer != iPointer) && (i < iLength)) i++;
	if (tPointer[i].iPointer == iPointer)
	   tPointer[i].num_free++;
        else  printf("free offlin %x\n", iPointer);         
     } /* end if (strstr(ligne,"free (") != NULL) */
} /* end while */

qsort(tPointer,iLength,sizeof(struct malloc_free),sort_address);
TotalSize = 0;
TotalFree = 0;
iSize = 0;
for (i=0; i<iLength; i++) {
  if (tPointer[i].num_alloc != tPointer[i].num_free){
    printf("malloc offlin %d %x (size %x), num_alloc=%d\n",
				i, tPointer[i].iPointer,
				tPointer[i].iSize,tPointer[i].num_alloc);
    iSize++;
    TotalSize += tPointer[i].iSize;
    }
  if (tPointer[i].num_alloc == tPointer[i].num_free) {
    for (j=i; j<iLength;j++)  {
	if (tPointer[j].num_alloc != tPointer[j].num_free) {
	     if ((tPointer[j+1].num_alloc != tPointer[j+1].num_free) && 
 		(tPointer[j+1].iPointer - 8 > tPointer[j].iPointer+iSize) &&
	        (j < iLength-1)) {	
                          printf("free hole at address (%x) size (%x)\n",
				tPointer[j].iPointer+tPointer[j].iSize,
				tPointer[j+1].iPointer-tPointer[j].iPointer
					- tPointer[j].iSize);
                          TotalFree += tPointer[j+1].iPointer-tPointer[j].iPointer-
						tPointer[j].iSize;
	                  }
             else {

                 printf("free hole at address (%x) size (%x),num_alloc = %d\n",
				tPointer[i].iPointer,
				tPointer[j].iPointer-tPointer[i].iPointer+16,
				tPointer[i].num_alloc);
                 TotalFree += tPointer[j].iPointer-tPointer[i].iPointer+16;
	         i = j-1;
                 }
             goto next;
             }   
       }
  }
next:;
}
printf("Total malloc non free = %d (pages %d)\n",TotalSize,
					(TotalSize+16*iSize+511)/512);
printf("Total memory allocated : %d pages",(iMaxPointer-iMinPointer+511)/512);
printf("Total free hole = %u (pages %u)\n", TotalFree,(TotalFree+511)/512);
}

    This produces a compilable file from the traces in TOTO.LOG(previous
    reply).
    Hope this helps.
    Philippe.
    
    PS: I logged a QAR for this malloc/free problem. If my remembering is
    correct, it was in OVMS VAX V6.1.
    
    
#define TABLEN	100000

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

struct malloc_free {
	int iPointer;
	int iSize;
	short num_alloc;
	short num_free;
};

int MAX(a, b)
int a, b;
{
if (a > b)
   return a;
else
   return b;
}

int main(argc, argv)
int argc;
char **argv;
{
FILE *fp;
FILE *fp1;
char ligne[100];
char ss[10];
int iPointer, iSize;
int iMinPointer,iMaxPointer;
int i,j, iLength;
struct malloc_free *tPointer;
unsigned int TotalFree,TotalSize;
char equal,open_bracket,close_bracket;

fp = fopen("toto.log", "r");
fp1 = fopen ("toto.c","w");
fprintf (fp1,"%s\n%s\n%s\n%s\n%s\n","#include <stdio.h>",
				"#include <stdlib.h>",
				"#include <string.h>",
                                "int main() {",
				"void lib$wait();");
iLength = iMaxPointer = 0;
tPointer = calloc(TABLEN,sizeof(struct malloc_free));
while ( fgets(&ligne[0],100,fp) != NULL) {
     if (strstr(ligne, "= malloc (") != NULL ) {
        sscanf (ligne,"%x %c %6s %c%x%c\n",&iPointer,&equal,ss,&open_bracket,&iSize,&close_bracket);
	i = 0;
	while ((tPointer[i].iPointer != iPointer) &&
		(tPointer[i].iPointer != 0)) {
	  i++;
	  if (i > TABLEN) {
	    printf("TABLEN trop petit.\n");
	    exit(0);
	  } /* end if */
	} /* end while */
	tPointer[i].iPointer= iPointer;
	tPointer[i].iSize=iSize;
	tPointer[i].num_alloc++;
	iLength = MAX(i,iLength);
     } /* end if (strstr(ligne,"= malloc (") != NULL) */

     if (strstr(ligne, "free (")  != NULL) {
        sscanf (ligne,"%4s %c%x%c\n",ss,&open_bracket,&iPointer,&close_bracket);
	i = 0;
	while ((tPointer[i].iPointer != iPointer) && (i < iLength)) i++;
	if (tPointer[i].iPointer == iPointer)
	   tPointer[i].num_free++;
        else  printf("free offlin %x\n", iPointer);         
     } /* end if (strstr(ligne,"free (") != NULL) */
} /* end while */
for (i=0; i<iLength;i++)
     fprintf(fp1,"%s%x%c\n","char *ptr",tPointer[i].iPointer,';');
rewind (fp);
while ( fgets(&ligne[0],100,fp) != NULL) {
     if (strstr(ligne, "= malloc (") != NULL ) {
        sscanf (ligne,"%x %c %6s %c%x%c\n",&iPointer,&equal,ss,&open_bracket,&iSize,&close_bracket);
        fprintf(fp1,"%s%x %c %6s%c%d%c%c\n","ptr",iPointer,equal,ss,open_bracket,iSize,close_bracket,';');
     } /* end if (strstr(ligne,"= malloc (") != NULL) */

     if (strstr(ligne, "free (")  != NULL) {
        sscanf (ligne,"%4s %c%x%c\n",ss,&open_bracket,&iPointer,&close_bracket);
        fprintf (fp1,"%4s %c%s%x%c%c\n",ss,open_bracket,"ptr",iPointer,close_bracket,';');
     } /* end if (strstr(ligne,"free (") != NULL) */
} /* end while */
fprintf (fp1,"lib$wait((float)10.0);\n");
fprintf (fp1,"%s\n","}");
}

:    Working Set and Virtual Page grew and No of lib$get_vm_page calls increased
:    as malloc() succeeded, but Working Set and Virtual Page never goes down
:    again even when free() succeded and lib$show_vm showed allocated memory size
:    was also decreasing.
:    
:    Is it normal and why?

   It's normal.  Underneath the "get_vm" services is an $expreg call, and
   the process address space is not (by default) deleted.

   This is one of the key reasons why I use the lookaside lists.

:    Sorry if this has to be posted on other notes.

   There are a number of previous discussions of this.

   Please log a QAR against OpenVMS, and one against DEC C, to get the
   documentation of the lib$get_vm and lib$free_vm calls, and of the
   malloc() and free() calls, updated to more clearly reflect this
   (expected) behaviour.