/*========================================================================
BLASTDAT: (C) Copyright 1992-2001, Fred Hutchinson Cancer Research Center
        blastdat.c  reads BLASTP output file
	    eg:  blastdat PS00011.hom PS00011.lis
			  or
		 blastdat PS00011.hom none

      If .lis file is provided, looks for those sequences in the .hom file.
      Always appends statistics to file named blastdat.dat.

-------------------------------------------------------------------------*/
/*   4/6/92  From fastodat.c
     9/16/93 Added ".tns" file which is list of true negative seqs found
>>>>>>>>>>>>>  End of matrix test project - blastp 1.2.9 <<<<<<<<<<<<<
   12/8/95   Modified for blastp version 1.4.7; has a space preceding
	     sequence name now.
    1/7/96   Added Pearson's E and ROC.
    1/9/96   Ignore sequences in block.
    1/10/96  Fix problem with blastdat not getting written.
    1/12/96  Increased NSCORE from 400 to 1000.
    1/13/96  Added split_names(), modified blastdat.dat
    1/27/96  Fix problem with E values, etc.
    2/ 4/96  Changed 99.5 to 99.9th percentile
---------------------------------------------------------------------------
    6/26/00  Modifications for blastall version 2.0.11
===========================================================================*/
#include "motifj.h"

#define NSCORE 1000		/* Maximum # of scores in .hom file */
#define MAXBUCKET 50		/* Maximum # of frequency buckets */
#define SWISS 307271		/* Number of sequences in database */
#define BIGINT 32000		/* Scores > than BIGINT are set = to it */

int read_hom();
int write_mis();
int write_fnd();
/*---------Routines from motmisc.o ----------*/
struct db_id *makedbid();
int get_ids();
struct split_name *split_names();

unsigned long LisSeq, FragSeq, BlkSeq;
char HomName[FNAMELEN];

/*======================================================================*/
int main(argc, argv)
int argc;
char *argv[];
{
   FILE *fhom, *flis, *fout;
   char homfile[FNAMELEN], lisfile[FNAMELEN];
   int nscores;
   struct db_id *ids;
   struct split_name *homsplit;

   printf("\nBLASTDAT: (C) Copyright 1992-2000, Fred Hutchinson Cancer Research Center");
   if (argc > 1) { strcpy(homfile, argv[1]);  }
   else
   {
      printf("\nEnter name of BLASTP results file: ");
      gets(homfile);
   }
   if ( (fhom=fopen(homfile, "r")) == NULL)
   {
      printf("\nCannot open file %s\n", homfile);
      exit(-1);
   }
   else printf("\nReading %s...", homfile);
   homsplit = split_names(homfile);
   strncpy(HomName, homfile + homsplit->dir_len, homsplit->name_len);
   HomName[homsplit->name_len] = '\0';

/*------------- arg 2:  .lis file -------------------------------*/
   if (argc > 2)
      strcpy(lisfile, argv[2]);
   else
   {
      printf("\nEnter name of file containing list of sequences: ");
      gets(lisfile);
   }
   if (!strlen(lisfile)) flis = NULL;
   else if ( (flis=fopen(lisfile, "r")) == NULL)
   {
      printf("\nCannot open file %s", lisfile);
      printf("\nNo list of sequences will be checked.");
   }
/*---------------  Get the sequences in the .lis file -----------------*/
   LisSeq = FragSeq = BlkSeq = 0;
   ids = makedbid();
   if (flis != NULL)
      printf("\n%ld IDs in %s\n", (LisSeq=get_ids(flis, ids)), lisfile);
   fclose(flis);

/*----------- Read the BLAST output file, write out list of true negs--*/
   strcpy(homfile, HomName); strcat(homfile, ".tns");
   if ( (fout=fopen(homfile, "wt")) == NULL)
   {
      printf("\nCannot open file %s\n", homfile);
      exit(-1);
   }
   fprintf(fout, "# True negative sequences for %s\n", HomName);
   nscores = read_hom(fhom, fout, ids);
   printf("\n%d total scores\n", nscores);
   fclose(fhom); fclose(fout);

/*--------------------Write out the missing sequences -----------------*/
   strcpy(homfile, HomName); strcat(homfile, ".mis");
   if ( (fout=fopen(homfile, "wt")) == NULL)
   {
      printf("\nCannot open file %s\n", homfile);
      exit(-1);
   }
   fprintf(fout, "# Sequences missed for %s\n", HomName);
   nscores = write_mis(fout, ids);
   printf("\n%d missed sequences written to %s", nscores, homfile);
   fclose(fout);
   /*-------------  Write sequences found to .fnd file -------------*/
   strcpy(homfile, HomName); strcat(homfile, ".fnd");
   if ( (fout=fopen(homfile, "wt")) == NULL)
   {
      printf("\nCannot open file %s\n", homfile);
      exit(-1);
   }
   fprintf(fout, "# Sequences found for %s\n", HomName);
   nscores = write_fnd(fout, ids);
   printf("\n%d found sequences written to %s", nscores, homfile);
   fclose(fout);

   printf("\n");
   exit(0);
}  /*  end of main */
/*======================================================================*/
int read_hom(fhom, ftns, ids)
FILE *fhom, *ftns;
struct db_id *ids;
{
   FILE *fout;			/* statistics */
   int tns, i, rank, tscore, lenid, len;
   int lfreq[MAXBUCKET], lminrank, lmaxrank;
   int iminrank, imaxrank, ntp999, done, nhits, tphits, tnhits;
   unsigned long swiss999;
   int found, freq[MAXBUCKET], minseq, maxseq, tps, lower, upper;
   int pearson, frag, inblocks;
   double dpvalue, roc, tn[NSCORE], tp[NSCORE], minscore, maxscore;
   char line[MAXLINE], id[22], *ptr, *ptr1;
   struct db_id *did;

   /*   Count the fragments & sequences in the .lis file  */
   LisSeq = FragSeq = BlkSeq = 0;
   did = ids->next;
   while (did != NULL)
   {
      if (!did->block) LisSeq ++;
      if (did->frag) FragSeq ++;
      if (did->block) BlkSeq ++;
      did = did->next;
   }

   /*---------Get the 99.9% score ------------------------------------*/
/*>>>>>>>>>>>Read SWISS from the output file <<<<<<<<<<<<<<<<<<<*/
   /* NOTE:  since tn[] & tp[] contain the probs, smaller is better   */
   swiss999 = (unsigned long) (SWISS-LisSeq-FragSeq-BlkSeq)*.001;
   printf("%ld true positives (%ld of them fragments), %ld in block\n",
          LisSeq, FragSeq, BlkSeq);
   printf(" Default 99.9th rank = %ld\n", swiss999);

   /* Initialize*/
   for (i=0; i<MAXBUCKET; i++)
   { freq[i] = lfreq[i] = 0; }
   tns = rank=0;
   minscore = 9.0; maxscore = 0.0;
   minseq = 9999; maxseq = tps = ntp999 = pearson = 0;
   lminrank = iminrank = 9999; lmaxrank = imaxrank = 0;
   tn[0] = tp[0] = tphits = tnhits = 0;
   frag = inblocks = 0;
   roc = 0.0;

/*>>>>>BLASTP specific:  get id, tscore, dpvalue, nhits */
   /*---- Copy first few lines -----*/
   done = YES;
   while (!feof(fhom) && fgets(line,MAXLINE,fhom) != NULL &&
          strstr(line, "Sequences producing") == NULL)
   {
      if (strstr(line, "Query") != NULL) done = NO;
      else if (strstr(line, "Searching") != NULL) done = YES;
      if (!done || strstr(line, "BLASTP") != NULL)  printf("%s", line);
   }
   fgets(line, MAXLINE, fhom);    /* next line is blank */
   /*---- Now process the results----------------------*/
   /* Results start after "Sequences producing" line & there is one
      result per line until the first blank line. Each line has a
      title for 66 chars, space, score, space, E-Value.  */
   done = NO;
   while (!done && !feof(fhom) && fgets(line,MAXLINE,fhom) != NULL)
   {
      if ((int) strlen(line) > 66)
      {
         /*------- Get the id -----------------*/
	 strncpy(id, line, SNAMELEN); id[SNAMELEN] = '\0';
         for (i=0; i<strlen(id); i++)
            if (id[i]==' ') id[i] = '\0';

	 /*----- Get the rank, score & nhits ------*/
	 rank++; 
         tscore = nhits = 0;
         dpvalue = 0.0;
         ptr=strtok(line+66, " \n\r");
         if (ptr != NULL)
         {
            tscore = atoi(ptr);
            ptr1 = strtok(NULL, " \n\r");
            if (ptr1 != NULL)
            {
               sscanf(ptr1, "%lg", &dpvalue);
            }
         }
/*<<<<<BLASTP specific */

	 /*----  now check to see if id is in the .lis file...  */
         /*----  id and did->entry may be different lengths, compare
                 only the basis of the shorter which should be a subset
                 of the longer  */
	 found = NO;
         lenid = strlen(id);
	 did = ids->next;
	 while (did != NULL)
	 {
            len = lenid;
            if (strlen(did->entry) < len) len = strlen(did->entry);
            if (!did->found &&
	        (strncmp(did->entry, id, len) == 0) )
	    {
               did->found = found = YES;
	       did->rank = rank;
	       did->score = tscore;
               did->pvalue = dpvalue;
               if (!did->block)
               {
	          i = (int) tscore/100;
	          if (i >= 0 && i < MAXBUCKET) ++lfreq[i];
	          tp[tps] = dpvalue;
	          tps++;  tphits += nhits;
	          if (rank < lminrank) lminrank = rank;
	          if (rank > lmaxrank) lmaxrank = rank;
	          if (tscore < minseq) minseq = tscore;
	          if (tscore > maxseq) maxseq = tscore;
	          printf("\nTP:Rank %.3d=%.20s, score=%d, P=%g",
			  rank, id, tscore, dpvalue);
	       }
               else if (did->block)
               {
                  inblocks++;
	          printf("\n BLOCK:Rank %.3d=%.20s, score=%d, P=%g",
			  rank, id, tscore, dpvalue);
               }
               if (did->frag)
               {
                  frag++;
/*	          printf("\n FRAGMENT:Rank %.3d=%.20s, score=%d, P=%g",
			  rank, id, tscore, dpvalue);
*/
               }
           }   /*  end of found  */
	    did = did->next;
	 } /* end of list entry */

         /*------  Write out true negatives --------------------*/
         if (!found)
         {
            fprintf(ftns, "%-20s", id);
            fprintf(ftns, "       P=%g", dpvalue);
            fprintf(ftns, "\n");
         }

         /*  Seqs not in list */
	 if (!found && tns < NSCORE)
	 {
	    i = (int) tscore/100;
	    if (i >= 0 && i < MAXBUCKET) ++freq[i];
	    tn[tns] = dpvalue;
	    if (tn[tns] < minscore) minscore = tn[tns];
	    if (tn[tns] > maxscore) maxscore = tn[tns];
	    if (rank < iminrank) iminrank = rank;
	    if (rank > imaxrank) imaxrank = rank;
	    if (tns==0)   printf("\n  1st TN score=%g", tn[0]);
	    else if ( ((tns+1)%100)==0 )
	        printf("\n  TN:%dth other score=%g",
                   tns+1, tn[tns]);
	    else if (tns == swiss999)
	        printf("\n  TN:%dth score(99.9)=%g", tns, tn[tns]);
	    /*  Print names of other top seqs */
	    if (rank <= LisSeq)
	    {
	        printf("\n  TN:Rank %.3d=%.20s, score=%d, P=%g",
			 rank, id, tscore, dpvalue);
	    }
	    tns++; tnhits += nhits;
            if (LisSeq > 0) roc += (double) tps / (double) LisSeq;
	 }  /* end of if !found */
      } /* end of if strlen(line) > 60 */
      else done=YES;
   }  /* end of while  for blastp output file */
   if (tns > 0) roc /= (double) tns;
   else roc = 1.0;

   printf("\n%d/%ld TP scores found, %d TN scores", tps, LisSeq, tns);
   printf("\n%d/%ld fragments found, %d/%ld of sequences in block found",
          frag, FragSeq, inblocks, BlkSeq);
/*
   printf("\nSummary of TP scores:  Maximum=%d, Median=%d, Minimum=%d",
             tp[0], tp[(int) tps/2], tp[tps-1]);
printf("\nSummary of TN scores:  Maximum=%d, Median=%d, Minimum=%d, 99th=%d",
      tn[0], tn[(int) tns/2], tn[tns-1],
      tn[(int) tns/100]);
*/
/*   printf("\nFrequency distribution of all scores:");
   printf("(*+=list, x/=other)");
   if (minseq < minscore) minscore = minseq;
   if (maxseq > maxscore) maxscore = maxseq;
   for (i=(int) minscore/100; i<= (int) maxscore/100; i++)
   {
      printf("\n %.4d-%.4d  %.4d  ", i*100, i*100+99, lfreq[i]+freq[i]);
      for (x=0; x< (int) lfreq[i]/5; x++)  printf("*");
      if ((freq[i]-5*x) > 0) printf("+");
      for (x=0; x< (int) freq[i]/5; x++)  printf("x");
      if ((freq[i]-5*x) > 0) printf("/");
   }
*/

/*=========================================================================*/
   /* NOTE:  since tn[] & tp[] contain the probs, smaller is better   */
/*---------Get the 99.9% score -------------------------------------------*/
   lower = swiss999;
   if (lower > tns-1) lower = tns-1;
   if (lower < 0) lower = 0;
   ntp999 = 0;		/* #tp seqs >= 99.9% of tn seqs */
   for (i=0; i < tps; i++)
      if (tp[i] <= tn[lower]) ntp999++;
   printf("\n%d/%ld true positive scores above ", ntp999, LisSeq);
   printf("99.9 percentile true negative score=%g", tn[lower]);

/*----- Pearson's E and ROC values -----------------------------*/
   pearson = LisSeq - tps;
   i = tps - 1;
   while (pearson >= 0 && pearson < tns &&
             i >= 0 && i < tps &&
             tn[pearson] < tp[i])
   { pearson++; i--; }
   printf("\n%d = Pearson equivalence number (%ld/%ld)",
              pearson, LisSeq - pearson, LisSeq);
   printf("\n%f = ROC area", roc);

/*----------Append statistics to blastdat.dat ----------------------------*/
   fout = NULL;
   if ((tns + tps + inblocks) > 0)	/* no stats if no results in file */
      if ( (fout=fopen("blastdat.dat", "a")) == NULL)
      {
	 printf("\nCannot open file blastdat.dat\n");
	 printf("\nNo statistics will be saved.");
      }
   if (fout != NULL)
   {
      fprintf(fout, "%s %d %ld %ld %ld %ld %d %d %d %d",
              HomName, tps+tns+inblocks, LisSeq+BlkSeq, BlkSeq, FragSeq, LisSeq,
              inblocks, frag, tps, tphits);
      /*  Median CI for z=2: order stat = (n+1)/2 - z*sqrt(n)/2.  */
      if (tps > 5)
      {
	 upper = (float) tps; upper=sqrt(upper);
	 upper += (float) (tps+1)/2;
      }
      else upper = (float) tps-1;
      if (upper < 0) upper= 0; if (upper > tps-1) upper=tps-1;
      /*  Info for seqs in the lis file (true positives)*/
/*
      fprintf(fout, " %d %d %d %d %d %d",
	lminrank, lmaxrank, tp[(int) upper],
	tp[tps-1], tp[(int) tps/2], tp[0]);
*/
      /*  Now add decile scores for seqs in the lis file */
/*
      tenth = (float) tps/10;
      for (i=9; i>=0; i--)
      {
	 x = (int) tenth/2 + i*tenth;
	 if (x<0) x = 0; if (x>tps-1) x=tps-1;
	 fprintf(fout, " %d", tp[x]); 
      }
*/
      /*  Info for seqs not in the lis file (true negatives) */
/*
      fprintf(fout, " %d %d %d %d %d %d",
	iminrank, imaxrank, tn[lower],
	tn[tns-1], tn[(int) tns/2], tn[0]);
*/
      fprintf(fout, " %d %d %d %d %f", tns, tnhits, ntp999,
        pearson, roc);
      fprintf(fout, "\n"); fclose(fout);
   }

   return(tns+tps+inblocks);
}  /* end of read_hom */
/*======================================================================*/
int write_mis(fmis, ids)
FILE *fmis;
struct db_id *ids;
{
   struct db_id *did;
   int nmiss;
 
   nmiss=0;
   did = ids->next;
   while (did != NULL)
   {
      if (did->rank <= 0 && !did->block)
      {
         fprintf(fmis, "%-20s", did->entry);
         if (strlen(did->ps)) fprintf(fmis, "  PS=%s", did->ps);
         if (did->len > 0) fprintf(fmis, "  LENGTH=%-6d", did->len);
         if (did->frag) fprintf(fmis, "  FRAGMENT");
         if (did->block) fprintf(fmis, "  BLOCK");
         if (did->lst) fprintf(fmis, "  LST");
         fprintf(fmis, "\n");
         nmiss++;
      }
      did = did->next;
   }

   return(nmiss);
}  /*  end of write_mis */
/*======================================================================*/
int write_fnd(ffnd, ids)
FILE *ffnd;
struct db_id *ids;
{
   struct db_id *did;
   int nfnd;
 
   nfnd=0;
   did = ids->next;
   while (did != NULL)
   {
      if (did->rank > 0 && !did->block)
      {
         fprintf(ffnd, "%-20s", did->entry);
         if (strlen(did->ps)) fprintf(ffnd, "  PS=%s", did->ps);
         if (did->len > 0) fprintf(ffnd, "  LENGTH=%-6d", did->len);
         if (did->frag) fprintf(ffnd, "  FRAGMENT");
         if (did->block) fprintf(ffnd, "  BLOCK");
         if (did->lst) fprintf(ffnd, "  LST");
         fprintf(ffnd, "       P=%g", did->pvalue);
         fprintf(ffnd, "\n");
         nfnd++;
      }
      did = did->next;
   }

   return(nfnd);
}  /*  end of write_fnd */
