[CF 6266] Better Performance for PostgreSQL with large INSERTs

This branch was automatically generated by a robot using patches from an
email thread registered at:

https://commitfest.postgresql.org/patch/6266

The branch will be overwritten each time a new patch version is posted to
the thread, and also periodically to check for bitrot caused by changes
on the master branch.

Patch(es): https://www.postgresql.org/message-id/CAFjYY+JTULmdQUJcBK-hDGRSWZy0e+RFHrCy3vp9J1pCQ18+Ew@mail.gmail.com
Author(s): Filip Januš

Author: Commitfest Bot

src/backend/libpq/pqcomm.c

@@ -112,19 +112,21 @@ static List *sock_paths = NIL;
 /*
  * Buffers for low-level I/O.
  *
- * The receive buffer is fixed size. Send buffer is usually 8k, but can be
- * enlarged by pq_putmessage_noblock() if the message doesn't fit otherwise.
+ * Both send and receive buffers are dynamically allocated. Send buffer is
+ * usually 8k, but can be enlarged by pq_putmessage_noblock() if the message
+ * doesn't fit otherwise. Receive buffer size is configurable via the
+ * pq_recv_buffer_size GUC parameter.
  */
 
 #define PQ_SEND_BUFFER_SIZE 8192
-#define PQ_RECV_BUFFER_SIZE 8192
 
 static char *PqSendBuffer;
 static int	PqSendBufferSize;	/* Size send buffer */
 static size_t PqSendPointer;	/* Next index to store a byte in PqSendBuffer */
 static size_t PqSendStart;		/* Next index to send a byte in PqSendBuffer */
 
-static char PqRecvBuffer[PQ_RECV_BUFFER_SIZE];
+static char *PqRecvBuffer;		/* Dynamically allocated receive buffer */
+static int	PqRecvBufferSize;	/* Size of receive buffer */
 static int	PqRecvPointer;		/* Next index to read a byte from PqRecvBuffer */
 static int	PqRecvLength;		/* End of data available in PqRecvBuffer */
 
@@ -278,6 +280,8 @@ pq_init(ClientSocket *client_sock)
 	/* initialize state variables */
 	PqSendBufferSize = PQ_SEND_BUFFER_SIZE;
 	PqSendBuffer = MemoryContextAlloc(TopMemoryContext, PqSendBufferSize);
+	PqRecvBufferSize = pq_recv_buffer_size * 1024;	/* GUC is in KB */
+	PqRecvBuffer = MemoryContextAlloc(TopMemoryContext, PqRecvBufferSize);
 	PqSendPointer = PqSendStart = PqRecvPointer = PqRecvLength = 0;
 	PqCommBusy = false;
 	PqCommReadingMsg = false;
@@ -898,15 +902,6 @@ pq_recvbuf(void)
 {
 	if (PqRecvPointer > 0)
 	{
-		if (PqRecvLength > PqRecvPointer)
-		{
-			/* still some unread data, left-justify it in the buffer */
-			memmove(PqRecvBuffer, PqRecvBuffer + PqRecvPointer,
-					PqRecvLength - PqRecvPointer);
-			PqRecvLength -= PqRecvPointer;
-			PqRecvPointer = 0;
-		}
-		else
 			PqRecvLength = PqRecvPointer = 0;
 	}
 
@@ -921,7 +916,7 @@ pq_recvbuf(void)
 		errno = 0;
 
 		r = secure_read(MyProcPort, PqRecvBuffer + PqRecvLength,
-						PQ_RECV_BUFFER_SIZE - PqRecvLength);
+						PqRecvBufferSize - PqRecvLength);
 
 		if (r < 0)
 		{

src/backend/utils/init/globals.c

@@ -165,3 +165,6 @@ int			notify_buffers = 16;
 int			serializable_buffers = 32;
 int			subtransaction_buffers = 0;
 int			transaction_buffers = 0;
+
+/* network buffer sizes */
+int			pq_recv_buffer_size = 8;

src/backend/utils/misc/guc_parameters.dat

@@ -2266,6 +2266,16 @@
   max => 'INT_MAX / 1000000',
 },
 
+{ name => 'pq_recv_buffer_size', type => 'int', context => 'PGC_BACKEND', group => 'RESOURCES_MEM',
+  short_desc => 'Sets the size of the network receive buffer for each backend.',
+  long_desc => 'Larger values can improve performance when receiving large messages, but consume more memory per connection.',
+  flags => 'GUC_UNIT_KB',
+  variable => 'pq_recv_buffer_size',
+  boot_val => '8',
+  min => '8',
+  max => 'MAX_KILOBYTES',
+},
+
 # Not for general use
 { name => 'pre_auth_delay', type => 'int', context => 'PGC_SIGHUP', group => 'DEVELOPER_OPTIONS',
   short_desc => 'Sets the amount of time to wait before authentication on connection startup.',

src/backend/utils/misc/postgresql.conf.sample

@@ -136,6 +136,7 @@
 #huge_page_size = 0                     # zero for system default
                                         # (change requires restart)
 #temp_buffers = 8MB                     # min 800kB
+#pq_recv_buffer_size = 8kB              # min 8kB
 #max_prepared_transactions = 0          # zero disables the feature
                                         # (change requires restart)
 # Caution: it is not advisable to set max_prepared_transactions nonzero unless

src/include/miscadmin.h

@@ -186,6 +186,7 @@ extern PGDLLIMPORT int notify_buffers;
 extern PGDLLIMPORT int serializable_buffers;
 extern PGDLLIMPORT int subtransaction_buffers;
 extern PGDLLIMPORT int transaction_buffers;
+extern PGDLLIMPORT int pq_recv_buffer_size;
 
 extern PGDLLIMPORT int MyProcPid;
 extern PGDLLIMPORT pg_time_t MyStartTime;