LP#1901932: (follow-up) update help in make_release
[evergreen-equinox.git] / build / tools / make_release
1 #!/bin/bash
2
3 GIT_ABS=`git rev-parse --show-toplevel`
4 GIT_BRANCH=`git rev-parse --abbrev-ref HEAD | sed 's|.*/||'`
5 HEADURLBASE="http://git.evergreen-ils.org/?p=Evergreen.git;a=shortlog;h=refs/heads/tags/"
6 HEADURL="$HEADURLBASE$GIT_BRANCH"
7
8 # Drop to the root of the checkout
9 cd $GIT_ABS
10
11 VERSION=AUTO # -v (version)
12 PREV_BRANCH=AUTO # -f (from)
13 PREV_VERSION=AUTO # -F (from version)
14 NO_UPGRADE=AUTO # -n
15 IS_PREVIEW=AUTO # -p
16 TAG_ONLY=NO # -t
17 BUILD_ONLY=NO # -b
18 UPGRADE_PREVIEW=NO # -r
19 SKIP_I18N=NO # -i
20 BUILD_BROWSER_CLIENT=NO # -c
21 SKIP_XUL=NO # -x
22 PG_HOST=NONE # -H
23 PG_USER=NONE # -U
24 PG_PASS=NONE # -P
25 PG_PORT=NONE # -O
26
27 # path to OpenSRF libraries
28 [ "$(which osrf_config)" ] && OSRF_JS_PATH="$(osrf_config --libdir)/javascript";
29
30
31 while getopts ":hv:f:F:nptbrij:cxH:D:U:P:O:" opt; do
32     case $opt in
33         v)
34             VERSION=$OPTARG
35         ;;
36         f)
37             PREV_BRANCH=$OPTARG
38         ;;
39         F)
40             PREV_VERSION=$OPTARG
41         ;;
42         n)
43             NO_UPGRADE=YES
44         ;;
45         r)
46             UPGRADE_PREVIEW=YES
47         ;;
48         i)
49             SKIP_I18N=YES
50         ;;
51         p)
52             IS_PREVIEW=YES
53         ;;
54         t)
55             TAG_ONLY=YES
56         ;;
57         b)
58             BUILD_ONLY=YES
59         ;;
60
61         j)
62             OSRF_JS_PATH="$OPTARG"
63         ;;
64         c)
65             BUILD_BROWSER_CLIENT=YES
66         ;;
67         x)
68             SKIP_XUL=YES
69         ;;
70         H)
71             PG_HOST="$OPTARG"
72         ;;
73         U)
74             PG_USER="$OPTARG"
75         ;;
76         P)
77             PG_PASS="$OPTARG"
78         ;;
79         O)
80             PG_PORT="$OPTARG"
81         ;;
82         \?)
83             echo "Invalid Option: -$OPTARG"
84             exit 1
85         ;;
86         :)
87             echo "-$OPTARG requires an argument."
88             exit 1
89         ;;
90         h)
91             echo "$0 [-v VERSION] [-f PREV_BRANCH | -t | -b] [-F PREV_VERSION] [-n] [-p]"
92             echo "   VERSION is auto-detected by default and is based on the currently checked out branch."
93             echo "   PREV_BRANCH is auto-detected by default but that isn't reliable. Include remote name!"
94             echo "   PREV_VERSION Is auto-detected by default and is based on the PREV_BRANCH's name."
95             echo "   -n specifies that you don't want an upgrade script to be auto-generated."
96             echo "   -p specifies that this is a preview build."
97             echo "   -t turns on tag only mode."
98             echo "   -b turns on build only mode."
99             echo "   -r prompt to preview upgrade SQL in editor before committing."
100             echo "   -i skip i18n; primarily useful for (quickly) testing this script."
101             echo "   -j opensrf javascript library path.  If osrf_config is found, the value derived from osrf_config --libdir."
102             echo "   -c build the browser client;  requires nodejs"
103             echo "   -x skip building the XUL client"
104             echo "   -H Postgres [H]ostname (optional)"
105             echo "   -D Postgres [D]atabase name (optional)"
106             echo "   -U Postgres [U]sername (optional)"
107             echo "   -P Postgres [P]assword (optional)"
108             echo "   -O Postgres p[O]rt"
109             echo ""
110             echo "   Postgres credentials are used to invoke the EXPERIMENTAL"
111             echo "   feature to update the Enhanced Concerto dataset."
112             echo ""
113             echo "   The provided Pg credentials needs to have permissions to create"
114             echo "   databases; using a Pg superuser will be most convenient."
115             echo ""
116             echo "   The commit created by the process to update the dataset"
117             echo "   should be thoroughly reviewed before producing the release."
118             echo ""
119             echo "   Generally, the update of the Enhanced Concerto dataset should"
120             echo "   be done and tested well before the release tarball is built; the main"
121             echo "   dependency is when the database schema becomes stable for the"
122             echo "   release, particularly changes that add or remove tables or"
123             echo "   columns."
124             echo ""
125             echo "   NOTE: -t and -b override PREV_BRANCH/PREV_VERSION, but -b overrides -t."
126             exit -1
127         ;;
128     esac
129 done
130
131 if [ -z "$OSRF_JS_PATH" ]; then
132     echo "Unable to find OpenSRF JavaScript library path.  Specify with -j";
133     exit 1;
134 fi;
135
136 if [ "$TAG_ONLY" == "YES" ]; then
137     PREV_BRANCH="TAG"
138 fi
139
140 if [ "$BUILD_ONLY" == "YES" ]; then
141     PREV_BRANCH="PACKAGE"
142 fi
143
144 if [ "$VERSION" == "AUTO" ]; then
145     # Auto-pick version based on branch name
146     echo AUTO VERSION
147     VERSION=`echo $GIT_BRANCH | sed 's/^rel_\([0-9]\+\)_\([0-9]\+\)_\([0-9]\+\)_\(.\+\)$/\1.\2.\3-\4/'`
148     VERSION=`echo $VERSION | sed 's/^rel_\([0-9]\+\)_\([0-9]\+\)_\([0-9]\+\)$/\1.\2.\3/'`
149     VERSION=`echo $VERSION | sed 's/^rel_\([0-9]\+\)_\([0-9]\+\)_\(.\+\)$/\1.\2-\3/'`
150     VERSION=`echo $VERSION | sed 's/^rel_\([0-9]\+\)_\([0-9]\+\)$/\1.\2/'`
151     if [ "$VERSION" = "$GIT_BRANCH" ]; then
152         echo "AUTO VERSION FAILED."
153         exit 1
154     fi
155 fi
156 SHORT_VERSION=`echo $VERSION | grep -o '^[0-9]\+\.[0-9]\+'`
157 echo "Version: $VERSION Short: $SHORT_VERSION"
158
159 # prep a couple alternate represenations
160 DASH_VERSION=`echo $VERSION | sed 's/\./-/g'`
161 SHORT_DASH_VERSION=`echo $SHORT_VERSION | sed 's/\./-/g'`
162 UNDER_VERSION=`echo $VERSION | sed -e 's/\./_/g;s/-/_/g'`
163
164 PREVIEW_TEXT=""
165
166 if [ "$IS_PREVIEW" == "YES" ]; then
167     PREVIEW_TEXT="previews/"
168 fi
169
170 # Release Preamble
171 # For adding into README for release builds
172 # The head is used to check if we have it already (no need to add it twice)
173 RELEASE_PREAMBLE_HEAD="Preamble: Getting an Evergreen official release tarball"
174 RELEASE_PREAMBLE=$( cat <<PREAMBLE
175 $RELEASE_PREAMBLE_HEAD
176 -------------------------------------------------------
177
178 To download and extract the source for the current release of Evergreen, issue
179 the following commands as the *user* Linux account:
180
181 [source, bash]
182 ------------------------------------------------------------------------------
183 wget -c http://evergreen-ils.org/downloads/${PREVIEW_TEXT}Evergreen-ILS-$VERSION.tar.gz
184 tar xzf Evergreen-ILS-$VERSION.tar.gz
185 ------------------------------------------------------------------------------
186
187 PREAMBLE
188 )
189
190 # This defines what the preamble comes before
191 RELEASE_PREAMBLE_BEFORE="Preamble: Developer instructions"
192
193 if [ "$PREV_BRANCH" != "TAG" -a "$PREV_BRANCH" != "PACKAGE" ]; then
194     if [ "$PREV_BRANCH" == "AUTO" ]; then
195         echo "AUTO PREVIOUS BRANCH"
196         PREV_BRANCH=`echo ${UNDER_VERSION%_*}`
197         PREV_BRANCH=`git branch -r | grep "rel_${PREV_BRANCH}_[^_]\+$" | sort -rV | head -n1`
198         PREV_BRANCH=`echo $PREV_BRANCH`
199         read -p "Does branch $PREV_BRANCH look like the previous version (y/n)?"
200         if [ "$REPLY" != "y" -a "$REPLY" != 'Y' ]; then
201             echo "Please specify the previous branch as second parameter. To continue auto-version, use AUTO as first parameter."
202             exit 1
203         fi
204     fi
205     git show $PREV_BRANCH &>/dev/null
206     if [ $? -ne 0 -o -z "$PREV_BRANCH" ]; then
207         echo "PREVIOUS VERSION COMMIT NOT FOUND";
208         exit 1
209     fi
210     if [ "$PREV_VERSION" == "AUTO" ]; then
211         echo "AUTO PREVIOUS VERSION"
212         PREV_BRANCH_END=`echo $PREV_BRANCH | sed 's|.*/||'`
213         PREV_VERSION=`echo $PREV_BRANCH_END | sed 's/^rel_\([0-9]\+\)_\([0-9]\+\)_\([0-9]\+\)_\(.\+\)$/\1.\2.\3-\4/'`
214         PREV_VERSION=`echo $PREV_VERSION | sed 's/^rel_\([0-9]\+\)_\([0-9]\+\)_\([0-9]\+\)$/\1.\2.\3/'`
215         PREV_VERSION=`echo $PREV_VERSION | sed 's/^rel_\([0-9]\+\)_\([0-9]\+\)_\(.\+\)$/\1.\2-\3/'`
216         PREV_VERSION=`echo $PREV_VERSION | sed 's/^rel_\([0-9]\+\)_\([0-9]\+\)$/\1.\2/'`
217         if [ "$PREV_VERSION" == "$PREV_BRANCH_END" ]; then
218             echo "AUTO PREVIOUS VERSION FAILED."
219             exit 1
220         fi
221     fi
222     echo "Previous Version: $PREV_VERSION"
223 else
224     echo "Tagging or packaging, no need for previous version"
225 fi
226
227 if [ "$PREV_BRANCH" != "PACKAGE" ]; then
228
229     echo "Applying to Application.pm - HEAD -> $DASH_VERSION"
230     echo "Alt: $SHORT_DASH_VERSION -> $DASH_VERSION"
231     sed -i -e "s/\"$SHORT_DASH_VERSION[^\"]*\"/\"$DASH_VERSION\"/" -e "s/\"HEAD\"/\"$DASH_VERSION\"/" $GIT_ABS/Open-ILS/src/perlmods/lib/OpenILS/Application.pm
232
233     if [ -f "$GIT_ABS/Open-ILS/xul/staff_client/windowssetup.nsi" ]; then
234         echo "Applying to windowssetup.nsi - Master -> $SHORT_VERSION"
235         sed -i "s/Master/$SHORT_VERSION/" $GIT_ABS/Open-ILS/xul/staff_client/windowssetup.nsi
236     fi
237
238     echo "Applying to README:"
239
240     if [ "$PREV_BRANCH" != "TAG" ]; then
241         if [ "$(grep "$RELEASE_PREAMBLE_HEAD" $GIT_ABS/README )" ]; then
242             echo "Updating old download links"
243             sed -i -e "s|\(previews/\)\?Evergreen-ILS-.*\.tar\.gz|${PREVIEW_TEXT}Evergreen-ILS-$VERSION.tar.gz|" $GIT_ABS/README
244             sed -i -e "s| Evergreen-ILS-.*\.tar\.gz| Evergreen-ILS-$VERSION.tar.gz|" $GIT_ABS/README
245         else
246             echo "Adding Download Preamble"
247             perl -pi -e "s|^|$RELEASE_PREAMBLE\n\n| if /$RELEASE_PREAMBLE_BEFORE/" $GIT_ABS/README
248         fi
249     fi
250
251     echo "Applying to configure.ac:"
252     echo "AC_INIT and AM_INIT_AUTOMAKE entries"
253     sed -i -e "s/^\(AC_INIT(Open-ILS, \)[^,]*,/\1$VERSION,/" -e "s/^\(AM_INIT_AUTOMAKE(\[[^]]*], \[\)[^]]*]/\1$VERSION]/" configure.ac
254
255     echo "Finding/updating old \$HeadURL\$ entries"
256     HEADURLBASE=`echo ${HEADURLBASE} | sed 's/\?/\\?/'`
257     for file in `grep -Rl --exclude=make_release "$HEADURLBASE" $GIT_ABS`
258     do
259         echo $file
260         sed -i "s|${HEADURLBASE}[A-Za-z0-9_]*|$HEADURL|" $file
261     done
262
263     echo "Applying \$HeadURL\$ - $HEADURL"
264     for file in `grep -Rl --exclude=make_release '\\$HeadURL\\$' $GIT_ABS`
265     do
266         echo $file
267         sed -i "s|\\\$HeadURL\\\$|$HEADURL|" $file
268     done
269
270     if [ "$PREV_BRANCH" = "TAG" ]; then
271         echo "Committing (but not pushing) the lot of it"
272         git commit -asm "Tagging version"
273         exit 0;
274     fi
275
276     echo "Applying version to 002.schema.config.sql:"
277     echo "config.upgrade_log entry for $VERSION"
278     sed -i -e "s/\(INSERT INTO config.upgrade_log (version[^)]*) VALUES ('\)[0-9]*\('.*;\).*/&\n\1$VERSION\2/" $GIT_ABS/Open-ILS/src/sql/Pg/002.schema.config.sql;
279
280     if [ "$NO_UPGRADE" == "AUTO" ]; then
281         echo "Checking for DB upgrade potential...."
282         git ls-tree --name-only $PREV_BRANCH -- Open-ILS/src/sql/Pg/upgrade/ | cut -d/ -f6 | cut -d. -f1 | sort > old_upgrades.txt
283         git ls-tree --name-only HEAD -- Open-ILS/src/sql/Pg/upgrade/ | cut -d/ -f6 | cut -d. -f1 | sort > new_upgrades.txt
284         UPGRADE_CHECK=`diff old_upgrades.txt new_upgrades.txt | grep '^>' | cut -d' ' -f2`
285         UPGRADE_FILE=Open-ILS/src/sql/Pg/version-upgrade/$PREV_VERSION-$VERSION-upgrade-db.sql
286         if [ -f "$UPGRADE_FILE" ]; then
287             echo "Upgrade script for $PREV_VERSION-$VERSION already exists. Skipping."
288             UPGRADE_FILE=""
289         else
290             echo "Building Upgrade Script."
291             echo "--Upgrade Script for $PREV_VERSION to $VERSION" > $UPGRADE_FILE
292             echo "\set eg_version '''$VERSION'''" >> $UPGRADE_FILE
293             echo "BEGIN;" >> $UPGRADE_FILE
294             grep "config.upgrade_log.*$VERSION" Open-ILS/src/sql/Pg/002.schema.config.sql >> $UPGRADE_FILE
295             for NUM in $UPGRADE_CHECK; do
296                 cat Open-ILS/src/sql/Pg/upgrade/$NUM.* 2>/dev/null | grep -v '^\s*\(BEGIN\|COMMIT\);\s*$' >> $UPGRADE_FILE
297             done;
298             echo "COMMIT;" >> $UPGRADE_FILE
299             #Add call to auditor.update_auditors() to keep auditor tables up to date
300             echo "" >> $UPGRADE_FILE
301             echo "-- Update auditor tables to catch changes to source tables." >> $UPGRADE_FILE
302             echo "--   Can be removed/skipped if there were no schema changes." >> $UPGRADE_FILE
303             echo "SELECT auditor.update_auditors();" >> $UPGRADE_FILE
304             MAYBE_DUPES=`grep -oP 'CREATE (OR REPLACE )?FUNCTION +\K[^ ]*(?= *\()' $UPGRADE_FILE | sort | grep -P '^(.*)\n\1$' | sort -u`
305             if [ -n "$MAYBE_DUPES" ]; then
306                 echo ""
307                 echo "The following functions may be needlessly duplicated in the upgrade script:"
308                 echo "$MAYBE_DUPES"
309                 echo ""
310                 echo "For reference, I am writing the list to maybe_dupes.txt"
311                 echo "$MAYBE_DUPES" > maybe_dupes.txt
312             fi
313             echo ""
314             if [ "$UPGRADE_PREVIEW" = "YES" ]; then
315                 read -p "Please manually check the upgrade file."
316                 ${EDITOR:-vi} $UPGRADE_FILE
317             fi;
318             git add $UPGRADE_FILE
319         fi
320     fi
321
322     grep -i -m 1 Signed-off-by ChangeLog &> /dev/null
323     if [ $? -ne 0 ]; then
324         echo "Building ChangeLog"
325         git log --cherry-pick --right-only --no-merges --pretty --summary --numstat $PREV_BRANCH...HEAD > $GIT_ABS/ChangeLog
326     else
327         echo "Not overwriting existing ChangeLog!"
328     fi
329
330     echo "Committing (but not pushing) the lot of it"
331     COMMIT_MESSAGE="Bumping version numbers and adding Changelog"
332     if [ -n "$UPGRADE_FILE" ]; then
333         COMMIT_MESSAGE="Bumping version numbers, adding Upgrade Script and Changelog"
334     fi
335     git commit -asm "$COMMIT_MESSAGE"
336     if [ "$PG_HOST" != "NONE" ]; then
337         echo "Attempting auto-upgrade for Enhanced Concerto Dataset"
338         $GIT_ABS/Open-ILS/src/support-scripts/make_concerto_from_evergreen_db.pl \
339         --db-host "$PG_HOST" --db-user "$PG_USER" --db-pass "$PG_PASS" --db-port "$PG_PORT" \
340         --output-folder "$GIT_ABS/Open-ILS/tests/datasets/sql/concerto_enhanced" \
341         --evergreen-repo "$GIT_ABS" \
342         --perform-upgrade \
343         --non-interactive
344
345         if [ $? -ne 0 ]; then
346             echo "Failed to auto-upgrade the Enhanced Concerto Dataset"
347             echo "Proceeding without it"
348             read -p "Press [Enter] key to continue"
349         else
350             echo "Committing Enhanced Concerto Dataset to git"
351             COMMIT_MESSAGE="Upgrade Enhanced Concerto Dataset"
352             git commit -asm "$COMMIT_MESSAGE"
353         fi
354     fi
355 fi
356
357 echo "Building release dump"
358 cd $GIT_ABS
359 mkdir -p ../release
360 git archive --prefix=Evergreen-ILS-$VERSION/ HEAD | (cd ../release && tar xf -)
361 cd ../release/Evergreen-ILS-$VERSION
362 if [ -f ./autogen.sh ]; then
363     echo "Running autogen"
364     ./autogen.sh
365 else
366     echo "Running autoreconf"
367     autoreconf -i
368 fi
369
370 if [ "$SKIP_I18N" == "NO" ]; then
371     cd build/i18n
372     echo "Building i18n"
373     make install_all_locales
374     echo "i18n Cleanup"
375     cd ..
376     rm -rf i18n
377     cd ..
378 fi;
379
380 echo "Installing Dojo"
381 if [ ! -f "../dojo.tgz" ]; then
382     wget http://evergreen-ils.org/downloads/dojo.tgz -O ../dojo.tgz
383 fi
384 tar xzf ../dojo.tgz -C Open-ILS/web/js/dojo/
385
386 # set version for later
387 cd Open-ILS/xul/staff_client
388 XULRUNNER_VERSION=`grep '^XULRUNNER_VERSION' Makefile.am`
389 XULRUNNER_VERSION=${XULRUNNER_VERSION##XULRUNNER_VERSION=}
390
391 echo "Prepping server download files"
392
393 if [ "$BUILD_BROWSER_CLIENT" == "YES" ]; then
394     # AngularJS staff client
395     cd ../../../
396     echo "Building AngularJS browser staff client"
397     cd Open-ILS/web/js/ui/default/staff/
398     npm install   # fetch build dependencies
399     npm run build-prod # copy to build dir and minify JS files
400     # npm cache is big and unnecessary in the final build. remove it.
401     rm -r node_modules 
402     cd ../../../../../ # Open-ILS dir
403
404     # OPAC dependencies
405     cd web/opac/deps
406     npm install # fetch dependencies
407     # and leave them in node_modules; there's no build step
408     cd ../../.. # Open-ILS dir
409
410     # Angular staff client
411     echo "Building Angular browser staff client"
412     cd src/eg2
413     npm install   # fetch build dependencies
414     ng build --configuration production --localize true
415     # npm cache is big and unnecessary in the final build. remove it.
416     rm -rf node_modules 
417     cd ../../../../ # release dir
418
419 else
420     echo "Skipping browser client build"
421     cd ../../../../
422 fi
423
424 tar czf Evergreen-ILS-$VERSION.tar.gz Evergreen-ILS-$VERSION/
425 md5sum Evergreen-ILS-$VERSION.tar.gz > Evergreen-ILS-$VERSION.tar.gz.md5
426 if [ $PREV_BRANCH != "PACKAGE" ]; then # We need to have tagged to do this ;)
427     cp Evergreen-ILS-$VERSION/ChangeLog ChangeLog-$PREV_VERSION-$VERSION
428 fi
429
430 if [ "$SKIP_XUL" = "NO" ]; then
431     echo "Running enough of configure to build staff client"
432     cd Evergreen-ILS-$VERSION/
433     ./configure --disable-core --disable-web --disable-updates --disable-apache-modules --disable-reporter
434
435     echo "Building Release Staff Clients"
436     cd Open-ILS/xul/staff_client
437
438     echo "Grabbing XULRunner (to avoid issues with version changes)"
439     wget http://ftp.mozilla.org/pub/mozilla.org/xulrunner/releases/$XULRUNNER_VERSION/runtimes/xulrunner-$XULRUNNER_VERSION.en-US.win32.zip
440     wget http://ftp.mozilla.org/pub/mozilla.org/xulrunner/releases/$XULRUNNER_VERSION/runtimes/xulrunner-$XULRUNNER_VERSION.en-US.linux-i686.tar.bz2
441     wget http://ftp.mozilla.org/pub/mozilla.org/xulrunner/releases/$XULRUNNER_VERSION/runtimes/xulrunner-$XULRUNNER_VERSION.en-US.linux-x86_64.tar.bz2
442
443     make rigrelease
444     make STAFF_CLIENT_STAMP_ID=rel_$UNDER_VERSION OPENSRF_JSLIBS="$OSRF_JS_PATH" build
445     make win-client
446     mv evergreen_staff_client_setup.exe ../../../../evergreen-setup-$VERSION.exe
447     make linux32-client
448     mv evergreen_staff_client_i686.tar.bz2 ../../../../evergreen-client-${VERSION}_i686.tar.bz2
449     make linux64-client
450     mv evergreen_staff_client_x86_64.tar.bz2 ../../../../evergreen-client-${VERSION}_x86_64.tar.bz2
451     cd ../../../../
452     md5sum evergreen-setup-$VERSION.exe > evergreen-setup-$VERSION.exe.md5
453     md5sum evergreen-client-${VERSION}_i686.tar.bz2 > evergreen-client-${VERSION}_i686.tar.bz2.md5
454     md5sum evergreen-client-${VERSION}_x86_64.tar.bz2 > evergreen-client-${VERSION}_x86_64.tar.bz2.md5
455 else
456     echo "Skipping XULRunner staff client build"
457 fi
458
459 echo "Removing build directory"
460 rm -rf Evergreen-ILS-$VERSION/
461
462 echo ""
463 echo "FOR YOU TODO:"
464 echo "* TEST the release"
465 if [ "$PREV_BRANCH" != "PACKAGE" ]; then
466     echo "* Push release branch"
467 fi
468 echo "* Upload files"
469 echo "* Send emails"