/* *************************************************************************** */
/* _tableio.c   	--  Fri Feb 20 1998
   A module for reading ASCII tables from files to Python lists.
   
   Important Note: the Python wrapper that calls readTable is
   responsible for making sure that things like '1', '5' and 'e' are
   not valid comment characters.  readTable is quite happy to treat
   '1.5e2' as a comment line.


   Copyright (C) 2000 Michael A. Miller <mmiller@debian.org>

   Time-stamp: <2000-08-09 17:09:14 miller>
   
   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public License as
   published by the Free Software Foundation; either version 2 of the
   License, or (at your option) any later version.
   
   This program is distributed in the hope that it will be useful, but
   WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   General Public License for more details.
   
   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
   USA. 

*/

/* ***************************************************************************
 Adapted from the TableIO package by Michael A. Miller. Thanks!

 Conrad Steenberg <conrad@srl.caltech.edu> 10 Aug 2000

*/

#include <stdio.h>
#include <string.h>
#include <glib.h>
#include <gtk/gtk.h>
#include <gtkextra/gtkextra.h>
#include <gdk/gdkkeysyms.h>
#include "python_main.h"
#define MAXLINE 32768

/* Some forward decls */
PyObject *
python_read_table_block(PyObject *self, PyObject *args, PyObject *keywds);
PyObject *
python_read_table_lines(PyObject *self, PyObject *args, PyObject *keywds);

PyObject *
python_read_table_block_string(PyObject *self, PyObject *args, PyObject *keywds);
PyObject *
python_read_table_lines_string(PyObject *self, PyObject *args, PyObject *keywds);


PyObject *python_read_build_array(GArray *array, gint x, gint y);
PyObject *python_read_build_list(GPtrArray *array, gint x, gint y);


GArray *
read_table(gchar *filename,gchar *commentchars,
          gchar *delim, gchar *blockstart,
          gint block, gint beginline, gint endline,
          gint *x,gint *y, GPtrArray **titles);

GPtrArray *
read_table_string(gchar *filename,gchar *commentchars,
          gchar *delim, gchar *blockstart,
          gint block, gint beginline, gint endline,
          gint *x,gint *y, GPtrArray **titles);


gchar err_msg[80]={""};

void g_ptr_array_free_strings(GPtrArray *array, gboolean free_segment, 
                              gboolean free_strings)
{ 
  gint i;

  if (free_strings)
   for (i=0; i<array->len;i++)
    g_free(array->pdata[i]);
  g_ptr_array_free(array,free_segment);
}

/* First the python language wrapper */

PyObject *
python_read_table_block(PyObject *self, PyObject *args, PyObject *keywds)
{ 
  gchar *comment="#",*delim=" \t\n",*blockstart="#";
  static gchar *kwlist[]={"filename","block","comment","delimiter",
  "blockstart",NULL};
  gchar *filename;           /* the name of the file from which to read */

  gint block=0,x,y;          /* Number of blocks in the file,counter */
  gint beginline=0,endline=0;/* Start and end line numbers to read */
  gint len,d1[1],d2[2];

  GArray *array;
 
  /* Make sure that I've got the arguments that I expect.  In this
     case a string ("s") that holds the filename */
  if (!PyArg_ParseTupleAndKeywords(args, keywds, "si|sss", kwlist,
                                   &filename, &block, &comment,
                                   &delim, &blockstart))
     return NULL;
  if (block<=0) block=1; /* guessing here */

  if (!filename || !strlen(filename))
  {
      PyErr_SetString(PyExc_IOError,"Empty filename specified");
      return NULL;
  }


  array=read_table(filename,comment,delim,blockstart,block,0,0,&x, &y,NULL);
  return (python_read_build_array(array,x,y));
}

PyObject *
python_read_table_lines(PyObject *self, PyObject *args, PyObject *keywds)
{ 
  gchar *comment="#",*delim=" \t\n";
  static gchar *kwlist[]={"filename","beginline","endline","comment","delimiter",NULL};
  gchar *filename;           /* the name of the file from which to read */

  gint block=0,x,y;          /* Number of blocks in the file,counter */
  gint beginline=0,endline=0;/* Start and end line numbers to read */
  gint len,d1[1],d2[2];

  GArray *array;
 
  /* Make sure that I've got the arguments that I expect.  In this
     case a string ("s") that holds the filename */
  if (!PyArg_ParseTupleAndKeywords(args, keywds, "sii|ss", kwlist,
                                   &filename, &beginline, &endline, &comment,
                                   &delim))
     return NULL;
  if (beginline<=0) beginline=1; /* guessing here */
  if (endline<0) endline=0;
  if (endline>0 && endline<beginline)
  {
      PyErr_SetString(PyExc_ValueError,"End line number must be greater than begin line.\nExcept 0 means read until EOF");
      return NULL;
  }   

  if (!filename || !strlen(filename))
  {
      PyErr_SetString(PyExc_IOError,"Empty filename specified");
      return NULL;
  }


  array=read_table(filename,comment,delim,NULL,0,beginline,endline,&x, &y,NULL);
  return (python_read_build_array(array,x,y));
}

PyObject *
python_read_table_block_string(PyObject *self, PyObject *args, PyObject *keywds)
{ 
  gchar *comment="#",*delim=" \t\n",*blockstart="#";
  static gchar *kwlist[]={"filename","block","comment","delimiter",
  "blockstart",NULL};
  gchar *filename;           /* the name of the file from which to read */

  gint block=0,x,y;          /* Number of blocks in the file,counter */
  gint beginline=0,endline=0;/* Start and end line numbers to read */
  gint len,d1[1],d2[2];

  GPtrArray *array;
 
  /* Make sure that I've got the arguments that I expect.  In this
     case a string ("s") that holds the filename */
  if (!PyArg_ParseTupleAndKeywords(args, keywds, "si|sss", kwlist,
                                   &filename, &block, &comment,
                                   &delim, &blockstart))
     return NULL;
  if (block<=0) block=1; /* guessing here */
  if (!filename || !strlen(filename))
  {
      PyErr_SetString(PyExc_IOError,"Empty filename specified");
      return NULL;
  }

  array=read_table_string(filename,comment,delim,blockstart,block,0,0,&x, &y,NULL);
  return (python_read_build_list(array,x,y));
}

PyObject *
python_read_table_lines_string(PyObject *self, PyObject *args, PyObject *keywds)
{ 
  gchar *comment="#",*delim=" \t\n";
  static gchar *kwlist[]={"filename","beginline","endline","comment","delimiter",NULL};
  gchar *filename;           /* the name of the file from which to read */

  gint block=0,x,y;          /* Number of blocks in the file,counter */
  gint beginline=0,endline=0;/* Start and end line numbers to read */
  gint len,d1[1],d2[2];

  GPtrArray *array;
 
  /* Make sure that I've got the arguments that I expect.  In this
     case a string ("s") that holds the filename */
  if (!PyArg_ParseTupleAndKeywords(args, keywds, "sii|ss", kwlist,
                                   &filename, &beginline, &endline, &comment,
                                   &delim))
     return NULL;
  if (beginline<=0) beginline=1; /* guessing here */
  if (endline<0) endline=0;
  if (endline>0 && endline<beginline)
  {
      PyErr_SetString(PyExc_ValueError,"End line number must be greater than begin line.\nExcept 0 means read until EOF");
      return NULL;
  }   
  if (!filename || !strlen(filename))
  {
      PyErr_SetString(PyExc_IOError,"Empty filename specified");
      return NULL;
  }


  array=read_table_string(filename,comment,delim,NULL,0,beginline,endline,&x, &y,NULL);
  return (python_read_build_list(array,x,y));
}


PyObject *python_read_build_array(GArray *array, gint x, gint y)
{  
  gint len,d1[1],d2[2];
  PyObject *parray;
  gdouble *temp_array;

  if (!array) /* Error ocurred */
  {
      PyErr_SetString(PyExc_IOError,err_msg);
      return NULL;
  }   

/* Now we construct a numeric python array */
  len=x*y;
  if (array->len<x*y)
  {
      temp_array=g_new0(gdouble,array->len-len);
      g_array_append_vals(array,(gconstpointer)temp_array,array->len-len);
  }
  if (x==0 || y==0)
      parray=Py_None;
  else if (x==1)
  {
      d1[0]=x;
#ifdef WITH_WARNINGS
#warning pointer array->data is never freed
#endif
      parray=PyArray_FromDimsAndData(1,d1,PyArray_DOUBLE,array->data);
  }
  else if (y==1)
  {
      d1[0]=y;
#ifdef WITH_WARNINGS
#warning pointer array->data is never freed
#endif
      parray=PyArray_FromDimsAndData(1,d1,PyArray_DOUBLE,array->data);
  }
  else
  {
      d2[0]=y;
      d2[1]=x;
#ifdef WITH_WARNINGS
#warning pointer array->data is never freed   
#endif
      parray=PyArray_FromDimsAndData(2,d2,PyArray_DOUBLE,array->data);
  }

  if (!parray) /* Error ocurred */
  {
      PyErr_SetString(PyExc_IOError,"Could not create array from data");
      return NULL;
  }   

  Py_INCREF(parray);
  return parray;
}

PyObject *python_read_build_list(GPtrArray *array, gint x, gint y)
{  
  gint len,d1[1],d2[2];
  PyObject *parray,*plist,*pstring,*str;
  gdouble *temp_array;
  gint i,j,count;
 
  if (!array) /* Error ocurred */
  {
      PyErr_SetString(PyExc_IOError,err_msg);
      return NULL;
  }   

  len=x*y;
  parray= PyList_New(y);
  count=0;
  if (parray)
   for (j=0;j<y;j++)
    { if (!(plist= PyList_New(x))) break;
      PyList_SET_ITEM (parray,j,plist);
      for (i=0;i<x;i++)
        if (count<len)
        { str=PyString_FromString ((gchar *)array->pdata[count++]);
          PyList_SET_ITEM (plist,i,str);
        }
    }
   
  if (!parray) /* Error ocurred */
  {
      PyErr_SetString(PyExc_IOError,"Could not create array from data");
      return NULL;
  }   

  Py_INCREF(parray);
  return parray;
}


/* Following the example of the original TableIO, we retaurn a 1D array */

GArray *
read_table(gchar *filename,gchar *commentchars,
           gchar *delim, gchar *blockstart,
           gint block, gint beginline, gint endline,
           gint *x,gint *y, GPtrArray **titles)
{
  FILE *infile;             /* the file pointer */
  GArray *status;           /* status flag */
                            
  GArray *array;            /* array to store data in */
  GPtrArray *my_titles;            /* array to store data in */

  gchar line[MAXLINE],dstr[80];      /* line read from the input file */
  gchar *ct;                /* character tokens for strtok */
  gchar *string=NULL;        /* holder for substrings */
  gchar *key;               /* python dictionary key */
  
  gint M;                    /* Number of data lines (rows) in the input file */
  gint N;                    /* Numer of data columns in the input file */
  gint n;                    /* used to check that N is constant */
  gint i,linenum;            /* counter */
  gint linesread=0;
  gchar *test1=(gchar *)1,*test2=(gchar *)1,*mark,*endptr;
  gdouble val;

  *x=0;
  *y=0;

  /* Check that the file exists and is readable */
  infile = fopen(filename, "r");
  if ( infile == NULL ) 
    {
      strncpy(err_msg,"Error opening input file",80);
      return NULL;
    }

  /* Determine how many rows and columns are here by skipping past all
     comment lines and parsing the first data line */
 if (block)
 for (i=0;i<block && test1;i++)
   { test1=fgets(line, sizeof(line), infile);

     do
      { test2=strpbrk(line,blockstart);
         if (!test2)
          { test1=fgets(line, sizeof(line), infile);
            mark=line;
          }
       } while (test1 && !test2);
    }
 else
  { for (i=0;i<beginline-1 && test1;i++)
      test1=fgets(line, sizeof(line), infile);
    linesread=i;
    test2=test1;
  }  
  if (!test1 || !test2)
  {
      strncpy(err_msg,"Wrong file format",80);
      return NULL;
  }

  /* Read next, non-comment line
  fgets(line, sizeof(line), infile);  */
  /* Now go through the line with strtok and see if I can figure out
     how many whitespace delimited fields there are.  
     Notes: 
     - The \n is necessary to take care of trailing whitespace.
     - At each step I check that I can convert the field to a float
       and stick it in a list.  If I can't, I quit.     
  */

  N = -1;
  ct = delim;
  linenum=beginline-1;
  
  while (!string)
  {   fgets(line, sizeof(line), infile);
      string = strtok( line, ct );
      linenum++;
  }
/* {
      mark=strpbrk(line,commentchars);
      if (mark) mark[0]='\0';
      if ( !mark || mark>line)
          string = strtok( line, ct );
      else
          if (!fgets(line, sizeof(line), infile))
          {
              strncpy(err_msg,"No valid data lines found",80);
              return NULL;
          };
  }
  /* Create an array */
  array=g_array_new (FALSE, FALSE, sizeof (gdouble));
  my_titles=g_ptr_array_new ();

  if ( string != NULL )
    { /* there is at least one field */
      val=strtod(string,&endptr);
      if (val==0 && string==endptr)
      {
          key=strdup(string);
          g_ptr_array_add(my_titles,(gpointer)key);
          status=(gpointer)1;
      }
      else
          status = g_array_append_val( array, val);
      N = 1;
      if ( !status ) /* This happens only when EOM */
	{
            strncpy(err_msg,"Error appending to list",80);
            g_array_free(array,TRUE);
            return NULL;

	}
      while (1) 
	{
	  string = strtok( NULL, ct );
	  if ( string != NULL ) 
	    {
              val=strtod(string,&endptr);
              if (val==0 && string==endptr)
              {
                  key=strdup(string);
                  g_ptr_array_add(my_titles,(gpointer)key);
                  status=(gpointer)1;
              }
              else
                status = g_array_append_val( array, val);
              N = N + 1;

              if ( !status )
              {
                   strncpy(err_msg,"Error appending to list",80);
                   g_array_free(array,TRUE);
                   return NULL;

              }
	    }
	  else 
	    {
	      break;
	    }
	}
    }


  /* Go through the rest of the file and fill the list */
  if (my_titles->len>0)
      M = 0;
  else M=1;

  if (block || (!block && (endline<=0 || (M<(endline-beginline+1)))))
  while (fgets(line, sizeof(line), infile) != NULL)
    { /* If this is not a comment line, treat it as data */
      linenum++;
      mark=strpbrk(line,commentchars);
      if (mark) mark[0]='\0';

      if ( !mark || mark>line )
	{
	  n = -1;
          string = strtok( line, ct );
	  if ( string != NULL )
	    { /* there is at least one field */
	      n = 1;
              val=atof(string);
              status = g_array_append_val( array, val);

              if ( !status )
              {
                    strncpy(err_msg,"Error appending to list",80);
                    g_array_free(array,TRUE);
                    return NULL;
	      }
	
	      while ( 1 ) 
		{
		  string = strtok( NULL, ct );

                  if ( string != NULL )
		    {
		      n = n + 1;
                      val=atof(string);
                      status = g_array_append_val( array, val);
		      if ( !status )
                      {
                           strncpy(err_msg,"Error appending to list",80);
                           g_array_free(array,TRUE);
                           return NULL;
                      }
		    }
		  else 
		    {
		      break;
		    }
                }
              if (my_titles->len>0 & M==0)
                  N=n;
              if (n!=N) break;
	      M = M + 1;
	    }
	  else break;
        }
      else if (block) break; /* A commentchar starts a new block */
      if (!block && endline>0 && (linenum>=endline)) break;
    }

  /* Close the input file */
  if ( fclose(infile) )
  {
      strncpy(err_msg,"Error closing input file",80);
      g_array_free(array,TRUE);
      g_ptr_array_free_strings(my_titles,TRUE,TRUE);
      return NULL;
  }

  /* If no data was found, report an error and return NULL */
  if ( N == -1 )
  { /* An apparently empty file */
      strncpy(err_msg,"File is apparently empty",80);
      g_array_free(array,TRUE);
      g_ptr_array_free_strings(my_titles,TRUE,TRUE);
      return NULL;
  }

  *x=N;
  *y=M;

  if (titles) *titles=my_titles;
  else g_ptr_array_free_strings(my_titles,TRUE,TRUE);
  return array;
}

GPtrArray *
read_table_string(gchar *filename,gchar *commentchars,
                  gchar *delim, gchar *blockstart,
                  gint block, gint beginline, gint endline,
                  gint *x,gint *y, GPtrArray **titles)
{
  FILE *infile;             /* the file pointer */
  GArray *status;           /* status flag */
                            
  GPtrArray *array;            /* array to store data in */
  GPtrArray *my_titles;            /* array to store data in */

  gchar line[MAXLINE],dstr[80];      /* line read from the input file */
  gchar *ct;                /* character tokens for strtok */
  gchar *string=NULL;        /* holder for substrings */
  gchar *key;               /* python dictionary key */
  
  gint M;                    /* Number of data lines (rows) in the input file */
  gint N;                    /* Numer of data columns in the input file */
  gint n;                    /* used to check that N is constant */
  gint i,linenum;            /* counter */
  gint linesread=0;
  gchar *test1=(gchar *)1,*test2=(gchar *)1,*mark,*endptr;
  gdouble val;

  *x=0;
  *y=0;

  /* Check that the file exists and is readable */
  infile = fopen(filename, "r");
  if ( infile == NULL ) 
    {
      strncpy(err_msg,"Error opening input file",80);
      return NULL;
    }

  /* Determine how many rows and columns are here by skipping past all
     comment lines and parsing the first data line */
 if (block)
   for (i=0;i<block && test1;i++)
   { 
     test1=fgets(line, sizeof(line), infile);

     do
      { 
         test2=strpbrk(line,blockstart);
         if (!test2)
          { 
            test1=fgets(line, sizeof(line), infile);
            mark=line;
          }
      } while (test1 && !test2);
    }
 else
  { 
    for (i=0;i<beginline-1 && test1;i++)
        test1=fgets(line, sizeof(line), infile);
    while (test2 && test1) {
        test1=fgets(line, sizeof(line), infile);
        i++;
        test2=strpbrk(line,commentchars);
    }
    linesread=i;
    test2=test1;
  }
  if (!test1 || !test2)
  {
      strncpy(err_msg,"Wrong file format",80);
      return NULL;
  }

  /* Read next, non-comment line
  fgets(line, sizeof(line), infile);  */

  /* Now go through the line with strtok and see if I can figure out
     how many whitespace delimited fields there are.  
     Notes: 
     - The \n is necessary to take care of trailing whitespace.
     - At each step I check that I can convert the field to a float
       and stick it in a list.  If I can't, I quit.     
  */

  N = -1;
  ct = delim;
  linenum=beginline-1;

  if (line) string = strtok( line, ct );
  while (!string)
  {   fgets(line, sizeof(line), infile);
      string = strtok( line, ct );
  }

  /* Create an array */
  array=g_ptr_array_new ();
  my_titles=g_ptr_array_new ();

  if ( string != NULL )
    { /* there is at least one field */
          key=strdup(string);
          if (titles) g_ptr_array_add(my_titles,(gpointer)key);
          else g_ptr_array_add( array, key);

      N = 1;
      if ( !key ) /* This happens only when EOM */
	{
            strncpy(err_msg,"Error appending to list",80);
            g_ptr_array_free_strings(my_titles,TRUE,TRUE);
            return NULL;

	}
      while (1) 
	{
	  string = strtok( NULL, ct );
	  if ( string != NULL ) 
	    {
              key=strdup(string);
              if (titles) g_ptr_array_add(my_titles,(gpointer)key);
              else g_ptr_array_add( array, key);

              N = N + 1;

              if ( !key )
              {
                   strncpy(err_msg,"Error appending to list",80);
                   g_ptr_array_free_strings(my_titles,TRUE,TRUE);
                   return NULL;

              }
	    }
	  else 
	      break;
	}
    }


  /* Go through the rest of the file and fill the list */
  if (titles && my_titles->len>0)
      M = 0;
  else M=1;

  if (block || (!block && (endline<=0 || (M<(endline-beginline+1)))))
  while (fgets(line, sizeof(line), infile) != NULL)
    { /* If this is not a comment line, treat it as data */
      linenum++;
      mark=strpbrk(line,commentchars);
      if (mark) mark[0]='\0';

      if ( !mark || mark>line )
	{
	  n = -1;
          string = strtok( line, ct );
	  if ( string != NULL )
	    { /* there is at least one field */
	      n = 1;
	         key=strdup(string);
              g_ptr_array_add( array, key);

	
	      while ( 1 ) 
		{
		  string = strtok( NULL, ct );

              if ( string != NULL )
		    {
		      n = n + 1;
	           key=strdup(string);
                g_ptr_array_add( array, key);

		      if ( !key )
                      {
                           strncpy(err_msg,"Error appending to list",80);
                           g_ptr_array_free_strings(array,TRUE,TRUE);
                           return NULL;
                      }
		    }
		  else 
		      break;
            }
              if (titles && my_titles->len>0 && M==0)
                  N=n;
              if (n!=N) { /* put handling for shorter lines here */
                  break;
              }
	      M = M + 1;
	    }
	  else  if (block) break;
        }
      else if (block) break; /* A commentchar starts a new block */
      if (!block && endline>0 && (linenum>=endline)) break;
    }

  /* Close the input file */
  if ( fclose(infile) )
  {
      strncpy(err_msg,"Error closing input file",80);
      g_ptr_array_free_strings(array,TRUE,TRUE);
      g_ptr_array_free_strings(my_titles,TRUE,TRUE);
      return NULL;
  }

  /* If no data was found, report an error and return NULL */
  if ( N == -1 )
  { /* An apparently empty file */
      strncpy(err_msg,"File is apparently empty",80);
      g_ptr_array_free_strings(array,TRUE,TRUE);
      g_ptr_array_free_strings(my_titles,TRUE,TRUE);
      return NULL;
  }

  *x=N;
  *y=M;

  if (titles) *titles=my_titles;
  else g_ptr_array_free_strings(my_titles,FALSE,TRUE);
  return array;
}