automate release process

Release process now: run script/release that's all to remember. In interactive (default) mode it will guide through bumping version number, editing CHANGELOG, creating release commit and tag, answer `n` to "Upload to PyPI" question.

- script/release will push new tag to Github
- Github will invoke Travis build hook
- Travis will build and upload PyPI package, granted all tests pass

script/release considers these sources of actual version information:

- python3/httplib2/__init__.py:__version__ = 'foobar'
- git tag in -auto mode (on Travis)
- what user entered in interactive mode

Every commit in every branch gets uploaded to Test PyPI, e.g. https://test.pypi.org/project/httplib2/0.11.1.post1803270210/
Every tagged commit in master gets uploaded to Public PyPI.

https://github.com/httplib2/httplib2/pull/99

3 files changed, 322 insertions, 69 deletions

diff --git a/.travis.yml b/.travis.yml
index 66c662e..982dad3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,20 +1,6 @@
 sudo: false
 language: python
 
-matrix:
-  fast_finish: true
-  include:
-    - {env: test_group=package, python: 2.7}
-    - {env: test_group=package, python: 3.6}
-    - {env: test_group=pep8, python: 2.7}
-    - {env: test_group=pep8, python: 3.6}
-    - {python: 2.7}
-    - {python: 3.3}
-    - {python: 3.4}
-    - {python: 3.5}
-    - {python: 3.6}
-    - {python: pypy}
-
 cache:
   apt: true
   ccache: true
@@ -22,7 +8,80 @@ cache:
   directories:
     - $HOME/.cache
 
-install:
-  - pip install 'pip>=9.0' 'setuptools>=36.2' 'codecov>=2.0.15' -r requirements-test.txt
-script:
-  - script/test -sv
+env:
+  global:
+    - pip_install_common='pip>=9.0 setuptools>=36.2 wheel>=0.30'
+python:
+  - 2.7
+  - 3.3
+  - 3.4
+  - 3.5
+  - 3.6
+  - pypy
+matrix:
+  fast_finish: true
+install: pip install $pip_install_common 'codecov>=2.0.15' -r requirements-test.txt
+script: script/test -sv && codecov
+
+stages:
+  - test
+  - release
+jobs:
+  include:
+    - stage: test
+      env: _=py2-pep8
+      python: 2.7
+      install: pip install -r requirements-test.txt
+      script: test_group=pep8 script/test
+    - stage: test
+      env: _=py3-pep8
+      python: 3.6
+      install: pip install -r requirements-test.txt
+      script: test_group=pep8 script/test
+    - stage: test
+      env: _=py2-package
+      python: 2.7
+      install: pip install $pip_install_common
+      script: test_group=package script/test
+    - stage: test
+      env: _=py3-package
+      python: 3.6
+      install: pip install $pip_install_common
+      script: test_group=package script/test
+    - stage: release
+      env: _=pypi-upload-test
+      python: 3.6
+      install: pip install $pip_install_common
+      script: script/release -auto
+      deploy:
+        provider: pypi
+        server: https://test.pypi.org/legacy/
+        user: httplib2.release.test
+        password:
+          secure: "XN3oxobC+26UPiS+F1MvL4c6XtytejZ13SkLXCHfgVDPSASzKqF81CnR4EhsnbfZLvSgGHgSlfY5Jve5HF2VR9GzpJMc6wzcfkkeBg6PeRHuMppIqmuoq7BTw81SZL9X62/mju/vxXs2cHpVkwNTSE7W1JH1bVXPj86oAR9xXo9waRuXcyPGNuSqmOd1NPOMbFmeuz+HeArk2Fz7wwo6H5BJuXjOrEOHWD1rzeRchH901PBUrftm54Id2TIVMARW8jm1saQY2FtPWaBv2v/DJC1fKWMJpcNQ3mmcvrrTFC1IJ00dk9XJfqx5hnsRaergc0UvzHoOGEQKSMdg0PUAkvNohAoCf+3GddPkvk8MaZ+aQlijoK6wp93A5dfTxBVZqdhmEdheolbYiJPunzS60bWvaEv6/D15/xyMiwGamUmF1Tx7UIvvm/zj6tAOBWbNEgLRyvQ0qx2RE95GLtp+RXK4pT+Kig1+cof5hrWODuEl1SSLMBySaNLWO73IN9esZu0X1JS7svnROLRJCAvRjppJYswwCPziP+B8XQDeMrhIDMHNzdbtxOPpBAXpYUE764FkzaUTMsK83Q+ugE3Dx8xtrAzT4M0fdiFv+3QEhSUtfvWsLH9zS9wXC5Px9kPKU3FO8mdUyf7A0bIasvJLNcApDJigKjBukToOIsZVFok="
+        # TODO: sdist bdist_wheel
+        # but wheels don't roll well with our 2/3 split code base
+        distributions: "sdist"
+        skip_cleanup: true
+        on:
+          repo: httplib2/httplib2
+          all_branches: true
+    - stage: release
+      if: (branch = master) AND (tag IS present)
+      env: _=pypi-upload-public
+      python: 3.6
+      install: pip install $pip_install_common
+      script: script/release -auto
+      deploy:
+        provider: pypi
+        user: httplib2.release
+        password:
+          secure: "jZAyMFnmbhYChjsb3gRYfESWlio6pgmWEWBRxtBQXYZf+tzyKVISyHuyWkJvOVTP+lOpp2MTPZ2s1UgxGwfzZ/VE034Cz5iA/C6wafmgtSW+wK+KEJFPseHBBA7Gh4ReiAPi2a+i1UXdsJpFNhv36E9tbTq2sEinbisE2lSEQ0KHadjkc+6pvCjlyhmes7QyM5GviWYlWRNj2OIkT8SUuUcWQt7ZEl6kN82MoMHCaf1YxE/i4JUP3VLomWK3RLZJP356Y4IDkzlVhFU4MJ4ubNtoQ/ECM0uQ+nsHzO0k1uGWdF6mMTna7U5gLqUi9rfCK3bLMeVSo+TUCpjI7HkWDaBgVXGTe5dUMJCDfRgqeYa0GnriI74XYJu8NGjMLv30uO58t9E7VQGo2NrFRJDzKAIHANejWnpUPY3XgoN1rlrh52seMjaU2+jO40EC8HvIqeRRwPwhkqCSV2y+IZT2bOFp2nbMWhkUMsxIX7OXt+sy8GvK/ilMleEl7r0tnudsT7lGdnMwXlttI3CIAFGE7E+0zwnxNiMzQDzo+ILVR7ezrCK9M9xVYKGa3i8gkpSn0Fblnltgg7HaEI8YC3rMZe4iu1t0D6cZZUAAp2ZUo3NCJcZ35iUFBhlFvjVDbe2upJgU6GFgtDLjyzCJiKbz8qLRgMFYgT0CGr512e1jBo0="
+        # TODO: sdist bdist_wheel
+        # but wheels don't roll well with our 2/3 split code base
+        distributions: "sdist"
+        skip_cleanup: true
+        on:
+          repo: httplib2/httplib2
+          branch: master
+          tags: true
diff --git a/script/release b/script/release
index 76d3dc5..556fc84 100755
--- a/script/release
+++ b/script/release
@@ -6,25 +6,79 @@ version=
 version_next=
 
 main() {
+	local opt_auto=0
+	while [[ $# -gt 0 ]] ; do
+		case $1 in
+		-auto)
+			opt_auto=1
+			;;
+		*)
+			echo "Usage: $0 [-auto]" >&2
+			exit 0
+			;;
+		esac
+		shift
+	done
+	if [[ "$opt_auto" -eq 1 ]] ; then
+		auto_prepare_release
+	else
+		interactive
+	fi
+}
+
+auto_prepare_release() {
+	echo 'script/release: auto mode for CI, will check or modify version based on tag' >&2
+
+	assert_tree_clean
+
+	local is_tag=0
+	local version_tag=
+	if version_tag=$(git describe --candidates=0 --tags HEAD 2>/dev/null) ; then
+		is_tag=1
+		version_tag=${version_tag##v}
+		version_check "$version_tag"
+	fi
+
+	local last_tag=
+	local version_replace=
+	if [[ "$is_tag" -eq 0 ]] ; then
+		last_tag=$(git tag --sort=-version:refname |head -n1)
+		last_tag=${last_tag##v}
+		version_replace="${last_tag}.post$(date -u +%y%m%d%H%M)"
+		update_version "setup.py" "s/VERSION =.+/VERSION = '$version_replace'/"
+		update_version "python2/httplib2/__init__.py" "s/__version__ =.+/__version__ = '$version_replace'/"
+		update_version "python3/httplib2/__init__.py" "s/__version__ =.+/__version__ = '$version_replace'/"
+		version_check "$version_replace"
+	fi
+}
+
+interactive() {
+	echo 'script/release: interactive mode for creating new tagged releases with human assistance' >&2
+
 	local branch="${1-$(git symbolic-ref --short HEAD)}"
-	version="$(PYTHONPATH=$PWD/python2 python2 -c 'import httplib2; print(httplib2.__version__)')"
+	version="$(PYTHONPATH=$PWD/python3 python3 -c 'import httplib2; print(httplib2.__version__)')"
 	printf "\nbranch: %s httplib2.__version__: '%s'\n" $branch $version >&2
 
-	if [[ "$branch" != "master" ]]; then
+	if [[ "$branch" != "master" ]] ; then
 		echo "Must be on master" >&2
 		exit 1
 	fi
-	if [[ -n "$(git status --short -uall)" ]]; then
-		echo "Tree must be clean. git status:" >&2
-		echo "" >&2
-		git status --short -uall
-		echo "" >&2
-		exit 1
+	assert_tree_clean
+
+	last_commit_message=$(git show --format="%s" --no-patch HEAD)
+	expect_commit_message="v$version release"
+	if [[ "$last_commit_message" != "$expect_commit_message" ]] ; then
+		printf "Last commit message: '%s' expected: '%s'\n" "$last_commit_message" "$expect_commit_message" >&2
+		if confirm "Create release commit? [yN] " ; then
+			create_commit
+		elif ! confirm "Continue without proper release commit? [yN] " ; then
+			exit 1
+		fi
 	fi
 	confirm "Continue? [yN] " || exit 1
 
 	echo "Creating tag v$version" >&2
-	if ! git tag "v$version"; then
+	if ! git tag "v$version" ; then
 		echo "git tag failed " >&2
 		confirm "Continue still? [yN] " || exit 1
 	fi
@@ -42,13 +96,137 @@ main() {
     fi
     $venv/bin/python setup.py sdist
 
-	if confirm "Upload to PyPi? [Yn] "; then
+	if confirm "Upload to PyPI? Use in special situation, normally CI (Travis) will upload to PyPI. [yN] " ; then
 		$venv/bin/twine upload dist/* || exit 1
 	fi
 
 	git push --tags
 }
 
+create_commit() {
+	echo "" >&2
+	echo "Plan:" >&2
+	echo "1. bump version" >&2
+	echo "2. update CHANGELOG" >&2
+	echo "3. commit" >&2
+	echo "4. run bin/release again" >&2
+	echo "" >&2
+
+	bump_version
+	edit_news
+
+	git diff
+	confirm "Ready to commit? [Yn] " || exit 1
+	git commit -a -m "v$version_next release"
+
+	echo "Re-exec $0 to continue" >&2
+	exec $0
+}
+
+bump_version() {
+	local current=$version
+	echo "Current version: '$current'" >&2
+	echo -n "Enter next version (empty to abort): " >&2
+	read version_next
+	if [[ -z "$version_next" ]] ; then
+		exit 1
+	fi
+	echo "Next version:    '$version_next'" >&2
+
+	update_version "python3/httplib2/__init__.py" "s/__version__ =.+/__version__ = '$version_next'/"
+	update_version "python2/httplib2/__init__.py" "s/__version__ =.+/__version__ = '$version_next'/"
+	update_version "setup.py" "s/VERSION =.+/VERSION = '$version_next'/"
+
+	confirm "Confirm changes? [yN] " || exit 1
+}
+
+update_version() {
+	local path="$1"
+	local sed_expr="$2"
+		# sed -E --in-place='' -e "s/VERSION =.+/VERSION = '$version_replace'/" setup.py
+		# sed -E --in-place='' -e "s/__version__ =.+/__version__ = '$version_replace'/" python2/httplib2/__init__.py python3/httplib2/__init__.py
+	echo "Updating file '$path'" >&2
+	if ! sed -E --in-place='' -e "$sed_expr" "$path" ; then
+		echo "sed error $?" >&2
+		exit 1
+	fi
+	assert_modified "$path"
+	echo "" >&2
+}
+
+edit_news() {
+	echo "Changes since last release:" >&2
+	git log --format='%h   %an   %s' "v$version"^.. -- || exit 1
+	echo "" >&2
+
+	patch -p1 <<EOT
+diff a/CHANGELOG b/CHANGELOG
+--- a/CHANGELOG
++++ b/CHANGELOG
+@@ -0,0 +1,4 @@
++$version_next
++
++  EDIT HERE. Describe important changes and link to more information.
++
+EOT
+
+	local editor=$(which edit 2>/dev/null)
+	[[ -z "$editor" ]] && editor="$EDITOR"
+	if [[ -n "$editor" ]] ; then
+		if confirm "Open default editor for CHANGELOG? [Yn] " ; then
+			$editor CHANGELOG
+		else
+			confirm "Edit CHANGELOG manually and press any key"
+		fi
+	else
+		echo "Unable to determine default text editor." >&2
+		confirm "Edit CHANGELOG manually and press any key"
+	fi
+	echo "" >&2
+
+	assert_modified CHANGELOG
+
+	echo "" >&2
+	confirm "Confirm changes? [yN] " || exit 1
+}
+
+assert_modified() {
+	local path="$1"
+	if git diff --exit-code "$path" ; then
+		echo "File '$path' is not modified" >&2
+		exit 1
+	fi
+}
+
+assert_tree_clean() {
+	if [[ -n "$(git status --short -uall)" ]] ; then
+		echo "Tree must be clean. git status:" >&2
+		echo "" >&2
+		git status --short -uall
+		echo "" >&2
+		exit 1
+	fi
+}
+
+version_check() {
+	local need=$1
+	local version_setup=$(fgrep 'VERSION =' setup.py |tr -d " '" |cut -d\= -f2)
+	local version_py2=$(cd python2 ; python2 -Es -c 'import httplib2;print(httplib2.__version__)')
+	local version_py3=$(cd python3 ; python3 -Es -c 'import httplib2;print(httplib2.__version__)')
+	if [[ "$version_setup" != "$need" ]] ; then
+		echo "error: setup.py VERSION=$version_setup expected=$need" >&1
+		exit 1
+	fi
+	if [[ "$version_py2" != "$need" ]] ; then
+		echo "error: python2/httplib2/__init__.py:__version__=$version_py2 expected=$need" >&1
+		exit 1
+	fi
+	if [[ "$version_py3" != "$need" ]] ; then
+		echo "error: python3/httplib2/__init__.py:__version__=$version_py3 expected=$need" >&1
+		exit 1
+	fi
+}
+
 confirm() {
 	local reply
 	local prompt="$1"
@@ -56,7 +234,7 @@ confirm() {
 	echo "" >&2
 	rc=0
 	local default_y=" \[Yn\] $"
-	if [[ -z "$reply" ]] && [[ "$prompt" =~ $default_y ]]; then
+	if [[ -z "$reply" ]] && [[ "$prompt" =~ $default_y ]] ; then
 		reply="y"
 	fi
 	[[ "$reply" != "y" ]] && rc=1
diff --git a/script/test b/script/test
index 4517afc..a3dca92 100755
--- a/script/test
+++ b/script/test
@@ -12,49 +12,65 @@ test_flags=(
 	tests/
 )
 
-cd "$( dirname "${BASH_SOURCE[0]}" )/.."
-if [[ -n "${CONTINUOUS_INTEGRATION-}" ]] ; then
-	case "${test_group-}" in
-	pep8)
-		if [[ "${TRAVIS_PYTHON_VERSION}" = "2.7" ]] ; then
-			flake8 python2/
-		else
-			flake8 python3/ tests/
+main() {
+	cd "$( dirname "${BASH_SOURCE[0]}" )/.."
+	if [[ -n "${CONTINUOUS_INTEGRATION-}" ]] ; then
+		case "${test_group-}" in
+		pep8)
+			if [[ "${TRAVIS_PYTHON_VERSION}" = "2.7" ]] ; then
+				flake8 python2/
+			else
+				flake8 python3/ tests/
+			fi
+			;;
+		package)
+			# TODO: sdist bdist_wheel
+			# but wheels don't roll well with our 2/3 split code base
+			python setup.py sdist
+			install_check_version "pip"
+			;;
+		*)
+			pip install -e .
+			httplib2_test_still_run_skipped=1 pytest --fulltrace -k test_303 $@ tests/ || true
+			httplib2_test_still_run_skipped=1 pytest --fulltrace -k test_head_301 $@ tests/ || true
+			pytest --fulltrace ${test_flags[@]}
+			;;
+		esac
+	else
+		if [[ ! -d ./venv-27 ]] ; then
+			virtualenv --python=python2.7 ./venv-27
 		fi
-		;;
-	package)
-	    # TODO: sdist bdist_wheel
-	    # but wheels don't roll well with our 2/3 split code base
-		python setup.py sdist
-		pip install dist/httplib2*
-		;;
-	*)
-		pip install -e .
-		httplib2_test_still_run_skipped=1 pytest --fulltrace -k test_303 $@ tests/ || true
-		httplib2_test_still_run_skipped=1 pytest --fulltrace -k test_head_301 $@ tests/ || true
-		pytest --fulltrace ${test_flags[@]}
-		codecov --flags=$(echo $python |tr -d -- '-.')
-		;;
-	esac
-else
-	if [[ ! -d ./venv-27 ]] ; then
-		virtualenv --python=python2.7 ./venv-27
+		if [[ ! -d ./venv-36 ]] ; then
+			virtualenv --python=python3.6 ./venv-36
+		fi
+
 		./venv-27/bin/pip install -e . -r requirements-test.txt
-	fi
-	if [[ ! -d ./venv-36 ]] ; then
-		virtualenv --python=python3.6 ./venv-36
+		./venv-27/bin/pytest ${test_flags[@]}
 		./venv-36/bin/pip install -e . -r requirements-test.txt
-	fi
+		./venv-36/bin/pytest ${test_flags[@]}
 
-	./venv-27/bin/pytest ${test_flags[@]}
-	./venv-36/bin/pytest ${test_flags[@]}
+		# FIXME: too many errors
+		# ./venv-27/bin/flake8 python2/
+		# ./venv-36/bin/flake8 python3/ tests/
 
-	# FIXME: too many errors
-	# ./venv-27/bin/flake8 python2/
-	# ./venv-36/bin/flake8 python3/ tests/
+		# TODO: sdist bdist_wheel
+		# but wheels don't roll well with our 2/3 split code base
+		./venv-36/bin/python setup.py sdist
+		install_check_version "./venv-27/bin/pip"
+		install_check_version "./venv-36/bin/pip"
+	fi
+	rm -rf ./_httplib2_test_cache
+}
 
-    # TODO: sdist bdist_wheel
-    # but wheels don't roll well with our 2/3 split code base
-	./venv-36/bin/python setup.py sdist
-fi
-rm -rf ./_httplib2_test_cache
+install_check_version() {
+	local pip="$1"
+	$pip install dist/httplib2*
+	version_source=$(cd python3 ; python3 -Es -c 'import httplib2;print(httplib2.__version__)')
+	version_installed=$($pip show httplib2 |fgrep Version |cut -d' ' -f2)
+	if [[ "$version_source" != "$version_installed" ]] ; then
+		echo "error: installed package version=$version_installed does not match source=$version_source" >&2
+		exit 1
+	fi
+}
+
+main "$@"