pipelining introduced, version bumped.

CHANGES

@@ -1,5 +1,8 @@
+
 0.1.2 (2008-05-19): 
 	- final support for asn32, now with correct syntax for Juniper. 
+	- experimental 'pipelining' mode (flag -T), much faster when 
+	working with big as-set's. 
 
 0.1.1 (2008-05-16): 
 	- initial support for asn32 added (flag -3). By default it's off, 

bgpq3.c

@@ -17,6 +17,7 @@
 #include "sx_report.h"
 
 extern int debug_expander;
+extern int pipelining;
 
 int
 usage(int ecode)
@@ -33,6 +34,7 @@ usage(int ecode)
 	printf(" -l        : use specified name for generated access/prefix/.."
 		" list\n");
 	printf(" -P        : generate prefix-list (default)\n");
+	printf(" -T        : pipelining (experimental, faster mode)\n");
 	printf(" -S sources: use only specified sources (default:"
 		" RADB,RIPE,APNIC)\n");
 	printf("\n" PACKAGE_NAME " version: " PACKAGE_VERSION "\n");
@@ -93,7 +95,7 @@ main(int argc, char* argv[])
 	bgpq_expander_init(&expander,af);
 	expander.sources=getenv("IRRD_SOURCES");
 
-	while((c=getopt(argc,argv,"36dhS:Jf:l:W:PG:"))!=EOF) { 
+	while((c=getopt(argc,argv,"36dhS:Jf:l:W:PG:T"))!=EOF) { 
 	switch(c) { 
 		case '3': 
 			expander.asn32=1;
@@ -122,6 +124,8 @@ main(int argc, char* argv[])
 			break;
 		case 'l': expander.name=optarg;
 			break;
+		case 'T': pipelining=1;
+			break;
 		case 'S': expander.sources=optarg;
 			break;
 		case 'W': expander.aswidth=atoi(optarg);

bgpq3.h

@@ -16,6 +16,13 @@ typedef enum {
 	T_PREFIXLIST
 } bgpq_gen_t;
 
+struct bgpq_prequest { 
+	struct bgpq_prequest* next;
+	char request[128];
+	int (*callback)(char*, void*);
+	void *udata;
+};
+
 struct bgpq_expander { 
 	struct sx_radix_tree* tree;
 	struct sx_slentry* macroses;
@@ -28,8 +35,11 @@ struct bgpq_expander {
 	int identify;
 	unsigned char asn32;
 	unsigned char* asn32s[65536];
+	struct bgpq_prequest* firstpipe, *lastpipe;
+	int piped;
 };
 
+
 int bgpq_expander_init(struct bgpq_expander* b, int af);
 int bgpq_expander_add_asset(struct bgpq_expander* b, char* set);
 int bgpq_expander_add_as(struct bgpq_expander* b, char* as);

bgpq_expander.c

@@ -19,6 +19,7 @@
 #include "sx_report.h"
 
 int debug_expander=0;
+int pipelining=0;
 
 int
 bgpq_expander_init(struct bgpq_expander* b, int af)
@@ -230,7 +231,131 @@ bgpq_expand_ripe(FILE* f, int (*callback)(char*, void*), void* udata,
 	return 0;
 };
 
+int
+bgpq_pipeline(FILE* f, int (*callback)(char*, void*), void* udata,
+	char* fmt, ...)
+{ 
+	char request[128];
+	int ret;
+	struct bgpq_prequest* bp=NULL;
+	struct bgpq_expander* d=(struct bgpq_expander*)udata;
+	va_list ap;
+	va_start(ap,fmt);
+	vsnprintf(request,sizeof(request),fmt,ap);
+	va_end(ap);
 
+	SX_DEBUG(debug_expander,"expander: sending '%s'\n", request);
+
+	bp=malloc(sizeof(struct bgpq_prequest));
+	if(!bp) { 
+		sx_report(SX_FATAL,"Unable to allocate %i bytes: %s\n", 
+			sizeof(struct bgpq_prequest),strerror(errno));
+		exit(1);
+	};
+	memset(bp,0,sizeof(bp));
+
+	ret=fwrite(request,1,strlen(request),f);
+	fseek(f,0,SEEK_END);
+
+	if(ret!=strlen(request)) { 
+		sx_report(SX_FATAL,"Partial write to radb, only %i bytes written: %s\n",
+			ret,strerror(errno));
+		exit(1);
+	};
+
+	strlcpy(bp->request,request,sizeof(bp->request));
+	bp->callback=callback;
+	bp->udata=udata;
+
+	if(d->lastpipe) { 
+		d->lastpipe->next=bp;
+		d->lastpipe=bp;
+	} else { 
+		d->firstpipe=d->lastpipe=bp;
+	};
+	d->piped++;
+
+	return 0;
+};
+
+int
+bgpq_pipeline_dequeue(FILE* f, struct bgpq_expander* b)
+{ 
+	while(b->piped>0) { 
+		char request[128];
+		struct bgpq_prequest* pipe;
+		memset(request,0,sizeof(request));
+		if(!fgets(request,sizeof(request),f)) { 
+			if(ferror(f)) { 
+				sx_report(SX_FATAL,"Error reading data from RADB: %s\n", 
+					strerror(errno));
+			} else { 
+				sx_report(SX_FATAL,"EOF from RADB\n");
+			};
+			exit(1);
+		};
+
+		if(request[0]=='A') { 
+			char* eon, *c;
+			unsigned long togot=strtoul(request+1,&eon,10);
+			char recvbuffer[togot+2];
+
+			if(eon && *eon!='\n') { 
+				sx_report(SX_ERROR,"A-code finished with wrong char '%c'(%s)\n",
+					*eon,request);
+				exit(1);
+			};
+			if(fgets(recvbuffer,togot+1,f)==NULL) { 
+				if(ferror(f)) { 
+					sx_report(SX_FATAL,"Error reading RADB: %s\n", 
+						strerror(errno));
+				} else { 
+					sx_report(SX_FATAL,"EOF from RADB\n");
+				};
+				exit(1);
+			};
+
+			for(c=recvbuffer; c<recvbuffer+togot;) { 
+				size_t spn=strcspn(c," \n");
+				if(spn) c[spn]=0;
+				if(c[0]==0) break;
+				if(b->firstpipe->callback) { 
+					b->firstpipe->callback(c,b->firstpipe->udata);
+				};
+				c+=spn+1;
+			};
+
+			/* Final code */
+			if(fgets(recvbuffer,togot,f)==NULL) { 
+				if(ferror(f)) { 
+					sx_report(SX_FATAL,"Error reading RADB: %s\n", 
+						strerror(errno));
+				} else { 
+					sx_report(SX_FATAL,"EOF from RADB\n");
+				};
+				exit(1);
+			};
+		} else if(request[0]=='C') { 
+			/* No data */
+		} else if(request[0]=='D') { 
+			/* .... */
+		} else if(request[0]=='E') { 
+			/* XXXXX */
+		} else if(request[0]=='F') { 
+			/* XXXXX */
+		} else { 
+			sx_report(SX_ERROR,"Wrong reply: %s to %s\n", request, 
+				b->firstpipe->request);
+		};
+
+		pipe=b->firstpipe;
+		b->firstpipe=b->firstpipe->next;
+		b->piped--;
+		free(pipe);
+				
+	};
+	return 0;
+};
 
 int
 bgpq_expand_radb(FILE* f, int (*callback)(char*, void*), void* udata,
@@ -409,17 +534,29 @@ bgpq_expand(struct bgpq_expander* b)
 									"-i origin as%u\r\n", i*8+j);
 
 						} else { 
-							if(k>0) 
-								bgpq_expand_radb(f,bgpq_expanded_prefix,b,
-									"!gas%u.%u\n", k, i*8+j);
-							else 
-								bgpq_expand_radb(f,bgpq_expanded_prefix,b,
-									"!gas%u\n", i*8+j);
+							if(!pipelining) { 
+								if(k>0) 
+									bgpq_expand_radb(f,bgpq_expanded_prefix,b,
+										"!gas%u.%u\n", k, i*8+j);
+								else 
+									bgpq_expand_radb(f,bgpq_expanded_prefix,b,
+										"!gas%u\n", i*8+j);
+							} else { 
+								if(k>0) 
+									bgpq_pipeline(f,bgpq_expanded_prefix,b,
+										"!gas%u.%u\n", k, i*8+j);
+								else 
+									bgpq_pipeline(f,bgpq_expanded_prefix,b,
+										"!gas%u\n", i*8+j);
+							};
 						};
 					};
 				};
 			};
 		};
+		if(pipelining) { 
+			bgpq_pipeline_dequeue(f,b);
+		};
 	};
 				
 	write(fd,"!q\n",3);

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.59 for bgpq3 0.1.1.
+# Generated by GNU Autoconf 2.59 for bgpq3 0.1.2.
 #
 # Report bugs to <snar@paranoia.ru>.
 #
@@ -269,8 +269,8 @@ SHELL=${CONFIG_SHELL-/bin/sh}
 # Identity of this package.
 PACKAGE_NAME='bgpq3'
 PACKAGE_TARNAME='bgpq3'
-PACKAGE_VERSION='0.1.1'
-PACKAGE_STRING='bgpq3 0.1.1'
+PACKAGE_VERSION='0.1.2'
+PACKAGE_STRING='bgpq3 0.1.2'
 PACKAGE_BUGREPORT='snar@paranoia.ru'
 
 ac_subst_vars='SHELL PATH_SEPARATOR PACKAGE_NAME PACKAGE_TARNAME PACKAGE_VERSION PACKAGE_STRING PACKAGE_BUGREPORT exec_prefix prefix program_transform_name bindir sbindir libexecdir datadir sysconfdir sharedstatedir localstatedir libdir includedir oldincludedir infodir mandir build_alias host_alias target_alias DEFS ECHO_C ECHO_N ECHO_T LIBS CC CFLAGS LDFLAGS CPPFLAGS ac_ct_CC EXEEXT OBJEXT LIBOBJS LTLIBOBJS'
@@ -738,7 +738,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures bgpq3 0.1.1 to adapt to many kinds of systems.
+\`configure' configures bgpq3 0.1.2 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -795,7 +795,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of bgpq3 0.1.1:";;
+     short | recursive ) echo "Configuration of bgpq3 0.1.2:";;
    esac
   cat <<\_ACEOF
 
@@ -906,7 +906,7 @@ fi
 test -n "$ac_init_help" && exit 0
 if $ac_init_version; then
   cat <<\_ACEOF
-bgpq3 configure 0.1.1
+bgpq3 configure 0.1.2
 generated by GNU Autoconf 2.59
 
 Copyright (C) 2003 Free Software Foundation, Inc.
@@ -920,7 +920,7 @@ cat >&5 <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by bgpq3 $as_me 0.1.1, which was
+It was created by bgpq3 $as_me 0.1.2, which was
 generated by GNU Autoconf 2.59.  Invocation command line was
 
   $ $0 $@
@@ -2702,7 +2702,7 @@ _ASBOX
 } >&5
 cat >&5 <<_CSEOF
 
-This file was extended by bgpq3 $as_me 0.1.1, which was
+This file was extended by bgpq3 $as_me 0.1.2, which was
 generated by GNU Autoconf 2.59.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -2762,7 +2762,7 @@ _ACEOF
 
 cat >>$CONFIG_STATUS <<_ACEOF
 ac_cs_version="\\
-bgpq3 config.status 0.1.1
+bgpq3 config.status 0.1.2
 configured by $0, generated by GNU Autoconf 2.59,
   with options \\"`echo "$ac_configure_args" | sed 's/[\\""\`\$]/\\\\&/g'`\\"
 

configure.in

@@ -1,4 +1,4 @@
-AC_INIT(bgpq3,0.1.1,snar@paranoia.ru)
+AC_INIT(bgpq3,0.1.2,snar@paranoia.ru)
 AC_CONFIG_HEADER(config.h)
 AC_PROG_CC