No blue-green for workers.

4 files changed, 282 insertions, 126 deletions

diff --git a/bootstrap-auditorreporter b/bootstrap-auditorreporter
new file mode 100755
index 0000000..ec593c2
--- /dev/null
+++ b/bootstrap-auditorreporter
@@ -0,0 +1,102 @@
+#!/bin/bash
+
+# Bootstrap the Taler setup for the user account that
+# is currently logged in.
+
+# The setup generated by this script is suitable for
+# blue-green deployments, where two users run
+# identical environments, and traffic can be switched
+# between them.
+
+# To create a demo setup, run
+# demo-blue$ ~/deployment/bootstrap-bluegreen demo
+# demo-green$ ~/deployment/bootstrap-bluegreen demo
+
+# NOTE: ENVNAME is also the name of the user who keeps
+# shared data in its home directory, so it must exist!
+# For demo/test ENVNAMEs, see {demo,test}.sh
+
+set -eu
+
+usage() {
+  echo "Usage: $0 ENVNAME"
+}
+
+if [ -z ${1+x} ]; then
+  usage
+  exit 1
+fi
+
+ENVNAME=$1
+source ~/envconfig
+
+case $ENVNAME in
+
+  test)
+    CURRENCY=TESTKUDOS
+    ;;
+  demo)
+    CURRENCY=KUDOS
+    ;;
+  *)
+    echo Invalid envname, only 'test' and 'demo'.
+    exit 1
+    ;;
+esac
+
+cd $HOME
+
+if ! test -d $HOME/gnunet; then
+  git clone git://gnunet.org/gnunet.git
+fi
+
+if ! test -d $HOME/libmicrohttpd; then
+  git clone git://gnunet.org/libmicrohttpd.git
+fi
+
+for component in exchange ; do
+  if ! test -d $HOME/$component; then
+    git clone git://git.taler.net/$component
+  fi
+done
+
+cat >$HOME/activate <<EOL
+#!/bin/bash
+
+function taler-config-generate () {
+  echo "Command disabled. Please use taler-deployment-config-generate instead."
+  return 1
+}
+function taler-exchange-keyup () {
+  echo "Command disabled. Please use taler-deployment-keyup instead."
+}
+
+
+export PATH="$HOME/deployment/bin:$HOME/local/bin:\$PATH"
+export PYTHONPATH="$HOME/local/lib/python3.5/site-packages"
+export TALER_CHECKDB="postgres:///talercheck-$USER"
+export TALER_CONFIG_CURRENCY=$CURRENCY
+
+# DEPRECATED ALIAS!
+export TALER_CONFIG_ENV=$ENVNAME
+
+export TALER_ENV_NAME=$ENVNAME
+export TALER_ENV_URL_INTRO="https://$ENVNAME.taler.net/"
+export TALER_ENV_URL_BANK="https://bank.$ENVNAME.taler.net/"
+export TALER_ENV_URL_MERCHANT_BLOG="https://shop.$ENVNAME.taler.net/"
+export TALER_ENV_URL_MERCHANT_DONATIONS="https://donations.$ENVNAME.taler.net/"
+export TALER_ENV_URL_MERCHANT_SURVEY="https://survey.$ENVNAME.taler.net/"
+export TALER_ENV_URL_AUDITOR="https://auditor.$ENVNAME.taler.net/"
+export TALER_ENV_URL_BACKOFFICE="https://backoffice.$ENVNAME.taler.net/"
+EOL
+
+mkdir -p "$HOME/sockets"
+mkdir -p "$HOME/stamps"
+mkdir -p "$HOME/taler-data"
+
+ln -sf ../deployment/taler-build/Makefile stamps/
+ln -sf ../deployment/taler-build/invalidate.sh stamps/
+
+if [[ ! -d "worker" ]]; then
+buildbot-worker create-worker $HOME/worker localhost:9989 "$ENVNAME-auditor-worker" "$ENVNAME-auditor-pass"
+fi
diff --git a/bootstrap-reservetopper b/bootstrap-reservetopper
new file mode 100755
index 0000000..d7fcc5f
--- /dev/null
+++ b/bootstrap-reservetopper
@@ -0,0 +1,102 @@
+#!/bin/bash
+
+# Bootstrap the Taler setup for the user account that
+# is currently logged in.
+
+# The setup generated by this script is suitable for
+# blue-green deployments, where two users run
+# identical environments, and traffic can be switched
+# between them.
+
+# To create a demo setup, run
+# demo-blue$ ~/deployment/bootstrap-bluegreen demo
+# demo-green$ ~/deployment/bootstrap-bluegreen demo
+
+# NOTE: ENVNAME is also the name of the user who keeps
+# shared data in its home directory, so it must exist!
+# For demo/test ENVNAMEs, see {demo,test}.sh
+
+set -eu
+
+usage() {
+  echo "Usage: $0 ENVNAME"
+}
+
+if [ -z ${1+x} ]; then
+  usage
+  exit 1
+fi
+
+ENVNAME=$1
+source ~/envconfig
+
+case $ENVNAME in
+
+  test)
+    CURRENCY=TESTKUDOS
+    ;;
+  demo)
+    CURRENCY=KUDOS
+    ;;
+  *)
+    echo Invalid envname, only 'test' and 'demo'.
+    exit 1
+    ;;
+esac
+
+cd $HOME
+
+if ! test -d $HOME/gnunet; then
+  git clone git://gnunet.org/gnunet.git
+fi
+
+if ! test -d $HOME/libmicrohttpd; then
+  git clone git://gnunet.org/libmicrohttpd.git
+fi
+
+for component in exchange ; do
+  if ! test -d $HOME/$component; then
+    git clone git://git.taler.net/$component
+  fi
+done
+
+cat >$HOME/activate <<EOL
+#!/bin/bash
+
+function taler-config-generate () {
+  echo "Command disabled. Please use taler-deployment-config-generate instead."
+  return 1
+}
+function taler-exchange-keyup () {
+  echo "Command disabled. Please use taler-deployment-keyup instead."
+}
+
+
+export PATH="$HOME/deployment/bin:$HOME/local/bin:\$PATH"
+export PYTHONPATH="$HOME/local/lib/python3.5/site-packages"
+export TALER_CHECKDB="postgres:///talercheck-$USER"
+export TALER_CONFIG_CURRENCY=$CURRENCY
+
+# DEPRECATED ALIAS!
+export TALER_CONFIG_ENV=$ENVNAME
+
+export TALER_ENV_NAME=$ENVNAME
+export TALER_ENV_URL_INTRO="https://$ENVNAME.taler.net/"
+export TALER_ENV_URL_BANK="https://bank.$ENVNAME.taler.net/"
+export TALER_ENV_URL_MERCHANT_BLOG="https://shop.$ENVNAME.taler.net/"
+export TALER_ENV_URL_MERCHANT_DONATIONS="https://donations.$ENVNAME.taler.net/"
+export TALER_ENV_URL_MERCHANT_SURVEY="https://survey.$ENVNAME.taler.net/"
+export TALER_ENV_URL_AUDITOR="https://auditor.$ENVNAME.taler.net/"
+export TALER_ENV_URL_BACKOFFICE="https://backoffice.$ENVNAME.taler.net/"
+EOL
+
+mkdir -p "$HOME/sockets"
+mkdir -p "$HOME/stamps"
+mkdir -p "$HOME/taler-data"
+
+ln -sf ../deployment/taler-build/Makefile stamps/
+ln -sf ../deployment/taler-build/invalidate.sh stamps/
+
+if [[ ! -d "worker" ]]; then
+buildbot-worker create-worker $HOME/worker localhost:9989 "$ENVNAME-topper-worker" "$ENVNAME-topper-pass"
+fi
diff --git a/buildbot/master.cfg b/buildbot/master.cfg
index 702049e..b1202cf 100644
--- a/buildbot/master.cfg
+++ b/buildbot/master.cfg
@@ -4,11 +4,11 @@ from buildbot.steps.source.git import Git
 from buildbot.steps.shell import ShellCommand
 from buildbot.plugins import (reporters, worker, changes, schedulers, util)
 
-# This is a sample buildmaster config file. It must be installed as
-# 'master.cfg' in your buildmaster's base directory.
-
-# This is the dictionary that the buildmaster pays attention to.
-# We also use a shorter alias to save typing.
+# This is a sample buildmaster config file. It must be
+# installed as 'master.cfg' in your buildmaster's base
+# directory.  This is the dictionary that the buildmaster
+# pays attention to.  We also use a shorter alias to save
+# typing.
 c = BuildmasterConfig = {}
 
 # Silence warning and allow very basic phoning home.
@@ -16,10 +16,10 @@ c["buildbotNetUsageData"] = "basic"
 
 ####### WORKERS
 
-# The 'workers' list defines the set of recognized workers. Each
-# element is a Worker object, specifying a unique worker name and
-# password.  The same worker name and password must be configured
-# on the worker.
+# The 'workers' list defines the set of recognized workers.
+# Each element is a Worker object, specifying a unique worker 
+# name and password.  The same worker name and password must
+# be configured on the worker.
 c["workers"] = [
 
     ##
@@ -42,46 +42,22 @@ c["workers"] = [
     worker.Worker("test-worker", "test-pass"),
 
     ##
-    # FIXME: who runs this?
-    #
-    worker.Worker("topper-worker", "topper-pass"),
-
-    ##
-    # This worker checks that all the services run under the
-    # 'demo' deployment are up&running.
-    worker.Worker("demo-worker", "demo-pass"),
-
-    ##
-    # This worker compiles the auditor reports for the test environment.
-    worker.Worker("test-auditor-worker", "test-auditor-pass"),
+    # Tip reserve toppers.
+    worker.Worker("test-topper-worker", "test-topper-pass"),
+    worker.Worker("demo-topper-worker", "demo-topper-pass"),
 
     ##
     # This worker compiles the auditor reports for the "green"
     # demo deployment.
-    worker.Worker("demo-auditor-worker-green", "demo-auditor-green-pass"),
+    worker.Worker("test-auditor-worker", "test-auditor-pass"),
+    worker.Worker("demo-auditor-worker", "test-auditor-pass"),
 
     ##
-    # This worker compiles the auditor reports for the "blue"
-    # demo deployment.
-    worker.Worker("demo-auditor-worker-blue", "demo-auditor-blue-pass"),
+    # This worker checks that all the services run under the
+    # 'demo' deployment are up&running.
+    worker.Worker("demo-worker", "demo-pass"),
 ]
 
-
-##
-# Maps usernames to workers.
-#
-def pick_map(buildername):
-
-    auditor_worker_map = {
-        "demo-green": "demo-auditor-worker-green",
-        "demo-blue": "demo-auditor-worker-blue"
-    }
-
-    meta_map = {"auditor-builder-demo": auditor_worker_map}
-
-    return meta_map.get(buildername)
-
-
 # 'protocols' contains information about protocols which master
 # will use for communicating with workers. You must define at
 # least 'port' option that workers could connect to your master
@@ -122,7 +98,9 @@ def twister_web_page(change):
 
 NIGHTLY_SCHEDULER = schedulers.Nightly(
     name="nightly-scheduler",
-    builderNames=["lcov-builder", "auditor-builder-demo", "auditor-builder"],
+    builderNames=["lcov-builder",
+                  "auditor-builder-demo",
+                  "auditor-builder-test"],
     branch="master",
     hour=6,
     minute=0
@@ -207,15 +185,13 @@ LINT_SCHEDULER = schedulers.SingleBranchScheduler(
 FORCE_SCHEDULER = schedulers.ForceScheduler(
     name="force-scheduler",
     builderNames=[
-        "lcov-builder", "auditor-builder", "auditor-builder-demo",
-        "test-builder", "doc-builder", "sites-builder",
+        "lcov-builder",
+        "auditor-builder-test",
+        "auditor-builder-demo",
+        "test-builder",
+        "doc-builder",
+        "sites-builder",
         "tip-reserve-topper-builder"
-
-        ##
-        # Rarely/never used, excluding.
-        # "wallet-builder",
-        # "selenium-builder",
-        # "debug-builder",
     ]
 )
 
@@ -431,17 +407,6 @@ BUILD_FACTORY.addStep(
     )
 )
 
-TESTSWITCH_FACTORY = util.BuildFactory()
-TESTSWITCH_FACTORY.addStep(
-    ShellCommand(
-        name="switch active party",
-        description="Switch to the party which was inactive.",
-        descriptionDone="Active party has been switched.",
-        command=["./switch.sh"],
-        workdir="../../deployment/buildbot"
-    )
-)
-
 SELENIUM_FACTORY = util.BuildFactory()
 SELENIUM_FACTORY.addStep(
     ShellCommand(
@@ -492,65 +457,44 @@ DEMO_SERVICES_CHECKER_FACTORY.addStep(
 )
 
 DEBUG_BUILDER = util.BuilderConfig(
-    name="debug-builder", workernames=["debug-worker"], factory=DEBUG_FACTORY
+    name="debug-builder",
+    workernames=["debug-worker"],
+    factory=DEBUG_FACTORY
 )
 
 LINT_BUILDER = util.BuilderConfig(
-    name="lint-builder", workernames=["lint-worker"], factory=LINT_FACTORY
+    name="lint-builder",
+    workernames=["lint-worker"],
+    factory=LINT_FACTORY
 )
 
 LCOV_BUILDER = util.BuilderConfig(
-    name="lcov-builder", workernames=["lcov-worker"], factory=LCOV_FACTORY
-)
-
-
-##
-# Helper function that picks the active deployment between
-# blue and green and schedules it as the next worker for producing
-# auditor reports.
-def pick_active_color(marker, builder, workers_list, build_request):
-
-    f = open(marker, "r")
-    if not f:
-        print("No active color in marker (%s)" % marker)
-        f.close()
-        return None
-    active = f.readline().rstrip()
-    print("Found active color: %s" % str(active))
-    aw = pick_map(builder.name).get(str(active))
-    for worker in workers_list:
-        if aw == worker.worker.workername:
-            f.close()
-            print("Scheduling worker: %s" % str(worker))
-            return worker
-
-    print("Couldn't find the active worker, aborting.")
-    f.close()
-    return None
-
-
-def pick_active_color_demo(builder, workers_list, build_request):
-    return pick_active_color(
-        "/home/demo/active", builder, workers_list, build_request
-    )
-
+    name="lcov-builder",
+    workernames=["lcov-worker"],
+    factory=LCOV_FACTORY
+)
 
 AUDITOR_BUILDER_TEST = util.BuilderConfig(
-    name="auditor-builder",
+    name="auditor-builder-test",
     workernames=["test-auditor-worker"],
     factory=AUDITOR_FACTORY
 )
 
 AUDITOR_BUILDER_DEMO = util.BuilderConfig(
     name="auditor-builder-demo",
-    nextWorker=pick_active_color_demo,
-    workernames=["demo-auditor-worker-blue", "demo-auditor-worker-green"],
+    workernames=["demo-auditor-worker"],
     factory=AUDITOR_FACTORY
 )
 
-TIP_RESERVE_TOPPER_BUILDER = util.BuilderConfig(
+TIP_RESERVE_TOPPER_BUILDER_DEMO = util.BuilderConfig(
     name="tip-reserve-topper-builder",
-    workernames=["topper-worker"],
+    workernames=["demo-topper-worker"],
+    factory=TIP_RESERVE_TOPPER_FACTORY
+)
+
+TIP_RESERVE_TOPPER_BUILDER_TEST = util.BuilderConfig(
+    name="tip-reserve-topper-builder",
+    workernames=["test-topper-worker"],
     factory=TIP_RESERVE_TOPPER_FACTORY
 )
 
@@ -593,11 +537,6 @@ c["builders"] = [
     DEMO_SERVICES_CHECKER_BUILDER,
     DOC_BUILDER,
     SITES_BUILDER,
-    #   Rarely/never used:
-    #   SELENIUM_BUILDER,
-    #   WALLET_BUILDER,
-    #   DEBUG_BUILDER,
-    #   LINT_BUILDER
 ]
 
 ####### BUILDBOT SERVICES
@@ -629,7 +568,6 @@ EMAIL = reporters.MailNotifier(
     mode=("change"),
     builders=(
         "lcov-builder",
-        "testswitcher-builder",
         "doc-builder",
         "test-builder",
         ##
@@ -654,40 +592,48 @@ c["buildbotURL"] = "https://buildbot.taler.net/"
 from taler_bb_userpass_db import USER_PASSWORD_DB
 
 BUILDER_LIST = [
-    "doc-builder", "lcov-builder", "auditor-builder", "auditor-builder-demo",
-    "demo-services-checker-builder", "tip-reserve-topper-builder",
-    "sites-builder", "test-builder"
-
-    ##
-    # Rarely/never used, excluding.
-    # "lint-builder",
-    # "wallet-builder",
-    # "selenium-builder"
+    "doc-builder",
+    "lcov-builder",
+    "auditor-builder",
+    "auditor-builder-test",
+    "auditor-builder-demo",
+    "demo-services-checker-builder",
+    "tip-reserve-topper-builder-demo",
+    "tip-reserve-topper-builder-test",
+    "sites-builder",
+    "test-builder"
 ]
 
 authz = util.Authz(
     allowRules=[
-        util.ForceBuildEndpointMatcher(role="admins", builder=b)
+        util.ForceBuildEndpointMatcher(role="admins",
+                                       builder=b)
         for b in BUILDER_LIST
     ] + [
-        util.StopBuildEndpointMatcher(role="admins", builder=b)
+        util.StopBuildEndpointMatcher(role="admins",
+                                      builder=b)
         for b in BUILDER_LIST
     ] + [
-        util.RebuildBuildEndpointMatcher(role="admins", builder=b)
+        util.RebuildBuildEndpointMatcher(role="admins",
+                                         builder=b)
         for b in BUILDER_LIST
     ] + [
-        util.ForceBuildEndpointMatcher(role="norole", builder=b)
+        util.ForceBuildEndpointMatcher(role="norole",
+                                       builder=b)
         for b in BUILDER_LIST
     ] + [
-        util.StopBuildEndpointMatcher(role="norole", builder=b)
+        util.StopBuildEndpointMatcher(role="norole",
+                                      builder=b)
         for b in BUILDER_LIST
     ] + [
-        util.RebuildBuildEndpointMatcher(role="norole", builder=b)
+        util.RebuildBuildEndpointMatcher(role="norole",
+                                         builder=b)
         for b in BUILDER_LIST
     ],
     roleMatchers=[
         util.RolesFromUsername(
-            roles=["admins"], usernames=["marcello", "florian", "christian"]
+            roles=["admins"],
+            usernames=["marcello", "florian", "christian"]
         )
     ]
 )
diff --git a/taler-build/invalidate.sh b/taler-build/invalidate.sh
index 5afa68a..0518a2f 100755
--- a/taler-build/invalidate.sh
+++ b/taler-build/invalidate.sh
@@ -1,9 +1,15 @@
 #!/bin/bash
 
 set -eu
-components="auditor deployment exchange merchant bank gnurl donations blog landing gnunet libmicrohttpd survey backoffice twister"
+COMPONENTS="auditor deployment exchange merchant bank gnurl donations blog landing gnunet libmicrohttpd survey backoffice twister"
+
+for component in $COMPONENTS ; do
+
+  # Not all the setups have all the repos!
+  if ! test -d $HOME/$component; then
+    continue
+  fi
 
-for component in  $components ; do
   cd $HOME/$component
   git fetch
   if git status -sb | grep behind; then