Split 'BufFile' routines out of fd.c into a new module, buffile.c. Extend

BufFile so that it handles multi-segment temporary files transparently.
This allows sorts and hashes to work with data exceeding 2Gig (or whatever
the local limit on file size is).  Change psort.c to use relative seeks
instead of absolute seeks for backwards scanning, so that it won't fail
when the data volume exceeds 2Gig.

5 files changed, 59 insertions, 41 deletions

diff --git a/src/include/executor/hashjoin.h b/src/include/executor/hashjoin.h
index 724a7ddd31..f501cd0365 100644
--- a/src/include/executor/hashjoin.h
+++ b/src/include/executor/hashjoin.h
@@ -6,7 +6,7 @@
  *
  * Copyright (c) 1994, Regents of the University of California
  *
- * $Id: hashjoin.h,v 1.14 1999/07/15 15:21:08 momjian Exp $
+ * $Id: hashjoin.h,v 1.15 1999/10/13 15:02:26 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -14,6 +14,7 @@
 #define HASHJOIN_H
 
 #include "access/htup.h"
+#include "storage/buffile.h"
 
 /* ----------------------------------------------------------------
  *				hash-join hash table structures
diff --git a/src/include/executor/nodeHashjoin.h b/src/include/executor/nodeHashjoin.h
index 2061ac7bdc..9d5390f79c 100644
--- a/src/include/executor/nodeHashjoin.h
+++ b/src/include/executor/nodeHashjoin.h
@@ -6,7 +6,7 @@
  *
  * Copyright (c) 1994, Regents of the University of California
  *
- * $Id: nodeHashjoin.h,v 1.15 1999/07/15 15:21:12 momjian Exp $
+ * $Id: nodeHashjoin.h,v 1.16 1999/10/13 15:02:26 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -14,6 +14,7 @@
 #define NODEHASHJOIN_H
 
 #include "nodes/plannodes.h"
+#include "storage/buffile.h"
 
 extern TupleTableSlot *ExecHashJoin(HashJoin *node);
 extern bool ExecInitHashJoin(HashJoin *node, EState *estate, Plan *parent);
diff --git a/src/include/storage/buffile.h b/src/include/storage/buffile.h
new file mode 100644
index 0000000000..2416d645cf
--- /dev/null
+++ b/src/include/storage/buffile.h
@@ -0,0 +1,47 @@
+/*-------------------------------------------------------------------------
+ *
+ * buffile.h
+ *	  Management of large buffered files, primarily temporary files.
+ *
+ * The BufFile routines provide a partial replacement for stdio atop
+ * virtual file descriptors managed by fd.c.  Currently they only support
+ * buffered access to a virtual file, without any of stdio's formatting
+ * features.  That's enough for immediate needs, but the set of facilities
+ * could be expanded if necessary.
+ *
+ * BufFile also supports working with temporary files that exceed the OS
+ * file size limit and/or the largest offset representable in an int.
+ * It might be better to split that out as a separately accessible module,
+ * but currently we have no need for oversize temp files without buffered
+ * access.
+ *
+ * Copyright (c) 1994, Regents of the University of California
+ *
+ * $Id: buffile.h,v 1.1 1999/10/13 15:02:32 tgl Exp $
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef BUFFILE_H
+#define BUFFILE_H
+
+#include "storage/fd.h"
+
+/* BufFile is an opaque type whose details are not known outside buffile.c. */
+
+typedef struct BufFile BufFile;
+
+/*
+ * prototypes for functions in buffile.c
+ */
+
+extern BufFile *BufFileCreateTemp(void);
+extern BufFile *BufFileCreate(File file);
+extern BufFile *BufFileReaccess(BufFile *file);
+extern void BufFileClose(BufFile *file);
+extern size_t BufFileRead(BufFile *file, void *ptr, size_t size);
+extern size_t BufFileWrite(BufFile *file, void *ptr, size_t size);
+extern int BufFileSeek(BufFile *file, int fileno, long offset, int whence);
+extern void BufFileTell(BufFile *file, int *fileno, long *offset);
+
+#endif	 /* BUFFILE_H */
diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h
index beb93bf699..42d1f46579 100644
--- a/src/include/storage/fd.h
+++ b/src/include/storage/fd.h
@@ -6,10 +6,11 @@
  *
  * Copyright (c) 1994, Regents of the University of California
  *
- * $Id: fd.h,v 1.17 1999/07/17 20:18:34 momjian Exp $
+ * $Id: fd.h,v 1.18 1999/10/13 15:02:32 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
+
 /*
  * calls:
  *
@@ -29,11 +30,6 @@
  * use FreeFile, not fclose, to close it.  AVOID using stdio for files
  * that you intend to hold open for any length of time, since there is
  * no way for them to share kernel file descriptors with other files.
- *
- * The BufFile routines provide a partial replacement for stdio.  Currently
- * they only support buffered access to a virtual file, without any of
- * stdio's formatting features.  That's enough for immediate needs, but
- * the set of facilities could be expanded if necessary.
  */
 #ifndef FD_H
 #define FD_H
@@ -46,25 +42,6 @@ typedef char *FileName;
 
 typedef int File;
 
-/* BufFile is an opaque type whose details are not known outside fd.c. */
-
-typedef struct BufFile BufFile;
-
-/* why is this here? fd.c doesn't want it ... */
-struct pgstat
-{								/* just the fields we need from stat
-								 * structure */
-	int			st_ino;
-	int			st_mode;
-	unsigned int st_size;
-	unsigned int st_sizehigh;	/* high order bits */
-/* 2^64 == 1.8 x 10^20 bytes */
-	int			st_uid;
-	int			st_atime_s;		/* just the seconds */
-	int			st_mtime_s;		/* since SysV and the new BSD both have */
-	int			st_ctime_s;		/* usec fields.. */
-};
-
 /*
  * prototypes for functions in fd.c
  */
@@ -78,24 +55,15 @@ extern void FileUnlink(File file);
 extern int	FileRead(File file, char *buffer, int amount);
 extern int	FileWrite(File file, char *buffer, int amount);
 extern long FileSeek(File file, long offset, int whence);
-extern int	FileTruncate(File file, int offset);
+extern int	FileTruncate(File file, long offset);
 extern int	FileSync(File file);
 
 /* Operations that allow use of regular stdio --- USE WITH CAUTION */
 extern FILE *AllocateFile(char *name, char *mode);
 extern void FreeFile(FILE *);
 
-/* Operations on BufFiles --- a very incomplete emulation of stdio
- * atop virtual Files...
- */
-extern BufFile *BufFileCreate(File file);
-extern void BufFileClose(BufFile *file);
-extern size_t BufFileRead(BufFile *file, void *ptr, size_t size);
-extern size_t BufFileWrite(BufFile *file, void *ptr, size_t size);
-extern long BufFileSeek(BufFile *file, long offset, int whence);
-
 /* Miscellaneous support routines */
-extern int	FileNameUnlink(char *filename);
+extern bool	ReleaseDataFile(void);
 extern void closeAllVfds(void);
 extern void AtEOXact_Files(void);
 extern int	pg_fsync(int fd);
diff --git a/src/include/utils/psort.h b/src/include/utils/psort.h
index 0deac02415..9a100bad0d 100644
--- a/src/include/utils/psort.h
+++ b/src/include/utils/psort.h
@@ -6,7 +6,7 @@
  *
  * Copyright (c) 1994, Regents of the University of California
  *
- * $Id: psort.h,v 1.21 1999/07/16 17:07:39 momjian Exp $
+ * $Id: psort.h,v 1.22 1999/10/13 15:02:28 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -15,7 +15,7 @@
 
 #include "access/relscan.h"
 #include "nodes/plannodes.h"
-#include "storage/fd.h"
+#include "storage/buffile.h"
 #include "utils/lselect.h"
 
 #define MAXTAPES		7		/* See Knuth Fig. 70, p273 */
@@ -57,7 +57,8 @@ typedef struct Psortstate
 	struct leftist *Tuples;
 
 	BufFile    *psort_grab_file;
-	long		psort_current;	/* could be file offset, or array index */
+	long		psort_current;	/* array index (only used if not tape) */
+	int			psort_saved_fileno;	/* upper bits of psort_saved, if tape */
 	long		psort_saved;	/* could be file offset, or array index */
 	bool		using_tape_files;
 	bool		all_fetched;	/* this is for cursors */