Hosting many domains as a secondary: parallelizing "nsdc update"
Stephane Bortzmeyer
bortzmeyer at nic.fr
Mon Jul 10 09:59:53 UTC 2006
On Fri, Jul 07, 2006 at 11:12:26AM +0200,
Stephane Bortzmeyer <bortzmeyer at nic.fr> wrote
a message of 214 lines which said:
> Therefore, I suggest the following solution:
Implemented today, we'll see if our Russian or Dutch colleagues scream
:-)
> Advices?
Here is the suggested patch against nsd 2.3.5. We would like to see it
integrated.
-------------- next part --------------
diff -r -u -d -N nsd-2.3.5.orig/Makefile.in nsd-2.3.5/Makefile.in
--- nsd-2.3.5.orig/Makefile.in 2006-05-08 11:26:05.000000000 +0200
+++ nsd-2.3.5/Makefile.in 2006-07-10 11:19:11.000000000 +0200
@@ -54,7 +54,7 @@
-e 's, at shell\@,$(SHELL),g' \
-e 's, at user\@,$(user),g'
-TARGETS = nsd zonec nsd-notify nsd-xfer nsdc.sh nsdc.conf.sample
+TARGETS = nsd zonec nsd-notify nsd-xfer nsdc.sh nsdc.conf.sample zones2make.py get-one-zone.sh
NSD_OBJECTS = \
answer.o \
@@ -158,6 +158,14 @@
rm -f nsdc.conf.sample
$(EDIT) nsdc.conf.sample.in > nsdc.conf.sample
+get-one-zone.sh: get-one-zone.sh.in
+ rm -f get-one-zone.sh
+ $(EDIT) get-one-zone.sh.in > get-one-zone.sh
+
+zones2make.py: zones2make.py.in
+ rm -f zones2make.py
+ $(EDIT) zones2make.py.in > zones2make.py
+
install: all
$(INSTALL) -d $(DESTDIR)$(sbindir)
$(INSTALL) -d $(DESTDIR)$(configdir)
@@ -166,6 +174,8 @@
$(INSTALL) nsd $(DESTDIR)$(sbindir)/nsd
$(INSTALL) zonec $(DESTDIR)$(sbindir)/zonec
$(INSTALL) nsdc.sh $(DESTDIR)$(sbindir)/nsdc
+ $(INSTALL) get-one-zone.sh $(DESTDIR)$(sbindir)/get-one-zone
+ $(INSTALL) zones2make.py $(DESTDIR)$(sbindir)/zones2make
$(INSTALL) nsd-notify $(DESTDIR)$(sbindir)/nsd-notify
$(INSTALL) nsd-xfer $(DESTDIR)$(sbindir)/nsd-xfer
$(INSTALL_DATA) nsd.8 $(DESTDIR)$(mandir)/man8
diff -r -u -d -N nsd-2.3.5.orig/get-one-zone.sh.in nsd-2.3.5/get-one-zone.sh.in
--- nsd-2.3.5.orig/get-one-zone.sh.in 1970-01-01 01:00:00.000000000 +0100
+++ nsd-2.3.5/get-one-zone.sh.in 2006-07-10 11:12:26.000000000 +0200
@@ -0,0 +1,108 @@
+#!/bin/sh
+#
+# get-one-zone: runs nsdxfer for *one* zone
+#
+# Copyright (c) 2001-2004, NLnet Labs. All rights reserved.
+#
+# See LICENSE for the license.
+#
+#
+
+# Optional configuration file for nsdc
+configfile="@configfile@"
+
+#
+# Default values in absense of ${configfile} (Usually ``/etc/nsd/nsdc.conf'')
+#
+prefix="@prefix@"
+exec_prefix="@exec_prefix@"
+
+sbindir="@sbindir@"
+zonesdir="@zonesdir@"
+flags=""
+dbfile="@dbfile@"
+zonesfile="@zonesfile@"
+keysdir="@configdir@/keys"
+notify="@sbindir@/nsd-notify"
+nsdxfer="@sbindir@/nsd-xfer"
+nsdxfer_flags=""
+pidfile="@pidfile@"
+lockfile="@dbfile at .lock"
+
+#
+# Read in configuration file if any
+#
+if [ -f ${configfile} ]
+then
+ . ${configfile}
+fi
+
+if [ "$3" = "" ]
+then
+ echo "Usage: $0 zone filename masters"
+ exit 1
+fi
+zone=$1
+file=$2
+masters=$3
+
+#
+# You sure heard this many times before: NO USER SERVICEABLE PARTS BELOW
+#
+if [ ! -x "${nsdxfer}" ]
+ then
+ echo "${nsdxfer} program is not available, aborting..."
+ exit 1
+fi
+
+
+# now get the serial number
+serial_opt=''
+if [ -e ${zonesdir}/$file ]; then
+ serial=`awk '/.*IN[ \t]+SOA.*\($/ { getline; print $1; exit }' ${zonesdir}/$file`
+ serial_opt="-s $serial"
+fi
+
+# take care of tsig info file if any
+# See bug #91 - move to ${zone} but be backward
+# compatible
+unset tsiginfoarg
+if [ -f "${keysdir}/${masters}.tsiginfo" ]
+ then
+ ln "${keysdir}/${masters}.tsiginfo" "${keysdir}/${masters}.tsiginfo.$$"
+ tsiginfoarg="-T ${keysdir}/${masters}.tsiginfo.$$"
+else
+ if [ -f "${keysdir}/${zone}.tsiginfo" ]
+ then
+ # the new way of doing things
+ ln "${keysdir}/${zone}.tsiginfo" "${keysdir}/${zone}.tsiginfo.$$"
+ tsiginfoarg="-T ${keysdir}/${zone}.tsiginfo.$$"
+ fi
+fi
+
+# AXFR to a temp file $file.axfr
+$nsdxfer $nsdxferflags -z $zone -f ${zonesdir}/$file.axfr ${tsiginfoarg} $serial_opt $masters
+if [ $? -eq 1 ]
+then
+ if [ -f ${zonesdir}/$file.axfr ]
+ then
+ # axfr succeeded
+ # test compile the zone to see what happens
+ ${sbindir}/zonec -o ${zone} -f ${zonesdir}/$file.axfr.db - < ${zonesdir}/$file.axfr 2>/dev/null
+ if [ $? -eq 1 ]
+ then
+ echo "Warning: AXFR of $zone did not compile"
+ rm -f ${zonesdir}/$file.axfr
+ else
+ # we succeed
+ mv -f ${zonesdir}/$file.axfr ${zonesdir}/$file
+ fi
+ rm -f ${zonesdir}/$file.axfr.db
+ else
+ echo "Warning: AXFR for $zone failed"
+ fi
+fi
+
+# clean up
+rm -f -- "${keysdir}"/*.tsiginfo.*
+
diff -r -u -d -N nsd-2.3.5.orig/nsdc.conf.sample.in nsd-2.3.5/nsdc.conf.sample.in
--- nsd-2.3.5.orig/nsdc.conf.sample.in 2006-02-07 14:44:17.000000000 +0100
+++ nsd-2.3.5/nsdc.conf.sample.in 2006-07-10 11:07:21.000000000 +0200
@@ -32,6 +32,18 @@
# Pathname of nsd-xfer binary
nsdxfer="@sbindir@/nsd-xfer"
+# Pathname of the Makefile for all zones. Only if USE_MAKE=yes, ignored otherwise
+makefile_name="@zonesdir@/Makefile.allzones"
+
+# Should we use make to update the zones?
+USE_MAKE=no
+
+# What make program to use? A parallel make is strongly recommened like GNU make
+MAKE="make"
+
+# How many instances of nsd-xfer should ne run simultaneously by make?
+MAKE_INSTANCES=30
+
# Flags to nsd-xfer. To set the socket's source address, use: "-a hostname" or "-a ip"
nsdxfer_flags=""
diff -r -u -d -N nsd-2.3.5.orig/nsdc.sh.in nsd-2.3.5/nsdc.sh.in
--- nsd-2.3.5.orig/nsdc.sh.in 2006-02-07 14:44:17.000000000 +0100
+++ nsd-2.3.5/nsdc.sh.in 2006-07-10 11:03:36.000000000 +0200
@@ -28,7 +28,11 @@
nsdxfer_flags=""
pidfile="@pidfile@"
lockfile="@dbfile at .lock"
+makefile_name="@zonesdir@/Makefile.allzones"
+USE_MAKE=no
+MAKE="make"
+MAKE_INSTANCES=30
ZONEC_VERBOSE=-v
#
@@ -96,70 +100,76 @@
lock
- # read the nsd.zones file
- while read zonekw zone file masterskw masters
- do
- if [ "X$zonekw" = "Xzone" -a "X$masterskw" = "Xmasters" ]
- then
+ if [ $USE_MAKE = "yes" ] || [ $USE_MAKE = "y" ]; then
+ ${MAKE} -j ${MAKE_INSTANCES} -f ${makefile_name} all
+ # TODO: do not rebuild uselessly, find a way to see if there was
+ # an actual zone transfer
+ rebuild="yes"
+ export $rebuild
+ else
+ # read the nsd.zones file
+ while read zonekw zone file masterskw masters
+ do
+ if [ "X$zonekw" = "Xzone" -a "X$masterskw" = "Xmasters" ]
+ then
# now get the serial number
- serial_opt=''
+ serial_opt=''
if [ -e ${zonesdir}/$file ]; then
- serial=`awk '/.*IN[ \t]+SOA.*\($/ { getline; print $1; exit }' ${zonesdir}/$file`
- serial_opt="-s $serial"
+ serial=`awk '/.*IN[ \t]+SOA.*\($/ { getline; print $1; exit }' ${zonesdir}/$file`
+ serial_opt="-s $serial"
fi
-
- # take care of tsig info file if any
+
+ # take care of tsig info file if any
# See bug #91 - move to ${zone} but be backward
# compatible
- unset tsiginfoarg
- if [ -f "${keysdir}/${masters}.tsiginfo" ]
- then
- ln "${keysdir}/${masters}.tsiginfo" "${keysdir}/${masters}.tsiginfo.$$"
- tsiginfoarg="-T ${keysdir}/${masters}.tsiginfo.$$"
+ unset tsiginfoarg
+ if [ -f "${keysdir}/${masters}.tsiginfo" ]
+ then
+ ln "${keysdir}/${masters}.tsiginfo" "${keysdir}/${masters}.tsiginfo.$$"
+ tsiginfoarg="-T ${keysdir}/${masters}.tsiginfo.$$"
else
- if [ -f "${keysdir}/${zone}.tsiginfo" ]
+ if [ -f "${keysdir}/${zone}.tsiginfo" ]
then
# the new way of doing things
- ln "${keysdir}/${zone}.tsiginfo" "${keysdir}/${zone}.tsiginfo.$$"
- tsiginfoarg="-T ${keysdir}/${zone}.tsiginfo.$$"
- fi
+ ln "${keysdir}/${zone}.tsiginfo" "${keysdir}/${zone}.tsiginfo.$$"
+ tsiginfoarg="-T ${keysdir}/${zone}.tsiginfo.$$"
+ fi
fi
-
+
# AXFR to a temp file $file.axfr
$nsdxfer $nsdxfer_flags -z $zone -f ${zonesdir}/$file.axfr ${tsiginfoarg} $serial_opt $masters
if [ $? -eq 1 ]
- then
- if [ -f ${zonesdir}/$file.axfr ]
+ then
+ if [ -f ${zonesdir}/$file.axfr ]
then
# axfr succeeded
# test compile the zone to see what happens
- ${sbindir}/zonec -o ${zone} -f ${zonesdir}/$file.axfr.db - < ${zonesdir}/$file.axfr 2>/dev/null
- if [ $? -eq 1 ]
- then
- echo "Warning: AXFR of $zone did not compile"
- rm -f ${zonesdir}/$file.axfr
- else
- # we succeed
- mv -f ${zonesdir}/$file.axfr ${zonesdir}/$file
- fi
- rm -f ${zonesdir}/$file.axfr.db
+ ${sbindir}/zonec -o ${zone} -f ${zonesdir}/$file.axfr.db - < ${zonesdir}/$file.axfr 2>/dev/null
+ if [ $? -eq 1 ]
+ then
+ echo "Warning: AXFR of $zone did not compile"
+ rm -f ${zonesdir}/$file.axfr
else
- echo "Warning: AXFR for $zone failed"
+ # we succeed
+ mv -f ${zonesdir}/$file.axfr ${zonesdir}/$file
fi
+ rm -f ${zonesdir}/$file.axfr.db
+ else
+ echo "Warning: AXFR for $zone failed"
+ fi
fi
-
- # Do we need to rebuild the database?
+ # Do we need to rebuild the database?
if [ ${zonesdir}/$file -nt ${dbfile} ]
- then
- echo "zone $zone needs rebuilding..."
- rebuild="yes"
- export $rebuild
+ then
+ echo "zone $zone needs rebuilding..."
+ rebuild="yes"
+ export $rebuild
fi
-
+ fi
+ done < $zonesfile
+ fi
# clean up
- rm -f -- "${keysdir}"/*.tsiginfo.*
- fi
- done < $zonesfile
+ rm -f -- "${keysdir}"/*.tsiginfo.*
# Wait for everybody to terminate
wait
diff -r -u -d -N nsd-2.3.5.orig/zones2make.py.in nsd-2.3.5/zones2make.py.in
--- nsd-2.3.5.orig/zones2make.py.in 1970-01-01 01:00:00.000000000 +0100
+++ nsd-2.3.5/zones2make.py.in 2006-07-10 11:15:47.000000000 +0200
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+# Converts a NSD zones file to a Makefile, to allow parallel updates.
+
+# Default values, may be overriden by config file
+config_file = "@configfile@"
+# The program to get *one* zone
+update_program = "@sbindir@/get-one-zone"
+# The location of the zones file
+zonesfile = "@zonesfile@"
+# The produced Makefile
+makefile_name = "@zonesdir@/Makefile.allzones"
+#
+zones2make_verbose = True
+
+import re, os, time
+
+comment = re.compile("^\s*;")
+config_comment = re.compile("^\s*#")
+zone_def = re.compile("^\s*zone\s+([a-z0-9\.-]+)\s+([^\s]+)\s+masters\s+(.+)$",
+ re.IGNORECASE)
+variable_def = re.compile("^\s*([a-z0-9_-]+)\s+=\s*([^\s]+)(#.*)?$",
+ re.IGNORECASE)
+temporary_makefile = makefile_name + ".tmp"
+
+config = open(config_file)
+for line in config.readlines():
+ if config_comment.search(line):
+ continue
+ match = variable_def.search(line)
+ if match:
+ variable = match.group(1)
+ value = match.group(2)
+ eval ("%s = %s" % (variable, value))
+config.close()
+
+zones = open(zonesfile)
+makefile = open(temporary_makefile, "w")
+makefile_content = ""
+first = True
+num = 0
+for line in zones.xreadlines():
+ if comment.search(line):
+ continue
+ match = zone_def.search(line)
+ if match:
+ zone = match.group(1)
+ file = match.group(2)
+ masters = match.group(3)
+ makefile_content = "%s\n%s:\n\t @%s %s %s %s\n" % (makefile_content,
+ zone, update_program,
+ zone, file, masters)
+ if first:
+ all = "all: %s" % (zone)
+ else:
+ all = "%s %s" % (all, zone)
+ first = False
+ num = num +1
+
+makefile.write(
+ "# AUTOMATICALLY GENERATED from %s on %s.\n# DO NOT EDIT!!!\n# %i domains\n%s\n\n" % \
+ (zonesfile,
+ time.strftime ("%Y-%B-%d %H:%M", time.localtime(time.time())),
+ num,
+ all))
+makefile.write(makefile_content)
+makefile.close()
+os.rename(temporary_makefile, makefile_name)
+zones.close()
More information about the nsd-users
mailing list