From fa0c4d2cb98ece0a7b156583d308ddf4370dc7d7 Mon Sep 17 00:00:00 2001 From: eshokrgozar Date: Thu, 16 Apr 2020 07:46:38 -0700 Subject: Switched the iri2uri import to a relative import Fixes an ImportError when using IronPython 2.7 --- python2/httplib2/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python2/httplib2/__init__.py b/python2/httplib2/__init__.py index 9057d1f..c8ef962 100644 --- a/python2/httplib2/__init__.py +++ b/python2/httplib2/__init__.py @@ -129,7 +129,7 @@ if ssl is None: _ssl_wrap_socket = _ssl_wrap_socket_unsupported if sys.version_info >= (2, 3): - from iri2uri import iri2uri + from .iri2uri import iri2uri else: def iri2uri(uri): -- cgit v1.2.3 From f5684876ef5e3b57c81f716c08b316fa36684f08 Mon Sep 17 00:00:00 2001 From: Sergey Shepelev Date: Wed, 22 Apr 2020 12:09:01 +0300 Subject: v0.17.3 release --- CHANGELOG | 5 +++++ python2/httplib2/__init__.py | 2 +- python3/httplib2/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index ba20b24..e148ba5 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,8 @@ +0.17.3 + + IronPython2.7: relative import iri2uri fixes ImportError + https://github.com/httplib2/httplib2/pull/163 + 0.17.2 python3 + debug + IPv6 disabled: https raised diff --git a/python2/httplib2/__init__.py b/python2/httplib2/__init__.py index c8ef962..7392e79 100644 --- a/python2/httplib2/__init__.py +++ b/python2/httplib2/__init__.py @@ -19,7 +19,7 @@ __contributors__ = [ "Alex Yu", ] __license__ = "MIT" -__version__ = '0.17.2' +__version__ = '0.17.3' import base64 import calendar diff --git a/python3/httplib2/__init__.py b/python3/httplib2/__init__.py index 135c6f6..19cd4d3 100644 --- a/python3/httplib2/__init__.py +++ b/python3/httplib2/__init__.py @@ -15,7 +15,7 @@ __contributors__ = [ "Alex Yu", ] __license__ = "MIT" -__version__ = '0.17.2' +__version__ = '0.17.3' import base64 import calendar diff --git a/setup.py b/setup.py index 9140845..eb32815 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ import setuptools.command.test import sys pkgdir = {"": "python%s" % sys.version_info[0]} -VERSION = '0.17.2' +VERSION = '0.17.3' # `python setup.py test` uses existing Python environment, no virtualenv, no pip. -- cgit v1.2.3 From fe3136ac369199abd9d6afd2d2a61a11da9e32ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20M=C3=A4rdian?= Date: Tue, 19 May 2020 15:32:25 +0200 Subject: Ship new test suite in source dist This makes the test suite available to Debian/Ubuntu/... It enables distributions to validate the quality of the shipped package. --- MANIFEST.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 12c4cc7..52ce781 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,6 @@ recursive-include python2 *.py *.txt recursive-include python3 *.py *.txt +graft test +graft tests include python2/httplib2/test/*.txt include requirements*.txt -- cgit v1.2.3 From 9413ffc973a2dc90abf787509ee82238345d5602 Mon Sep 17 00:00:00 2001 From: Sergey Shepelev Date: Tue, 19 May 2020 17:28:48 +0300 Subject: v0.17.4 release --- CHANGELOG | 5 +++++ python2/httplib2/__init__.py | 2 +- python3/httplib2/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index e148ba5..4f4ca13 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,8 @@ +0.17.4 + + Ship test suite in source dist + https://github.com/httplib2/httplib2/pull/168 + 0.17.3 IronPython2.7: relative import iri2uri fixes ImportError diff --git a/python2/httplib2/__init__.py b/python2/httplib2/__init__.py index 7392e79..97e06c1 100644 --- a/python2/httplib2/__init__.py +++ b/python2/httplib2/__init__.py @@ -19,7 +19,7 @@ __contributors__ = [ "Alex Yu", ] __license__ = "MIT" -__version__ = '0.17.3' +__version__ = '0.17.4' import base64 import calendar diff --git a/python3/httplib2/__init__.py b/python3/httplib2/__init__.py index 19cd4d3..8785cc1 100644 --- a/python3/httplib2/__init__.py +++ b/python3/httplib2/__init__.py @@ -15,7 +15,7 @@ __contributors__ = [ "Alex Yu", ] __license__ = "MIT" -__version__ = '0.17.3' +__version__ = '0.17.4' import base64 import calendar diff --git a/setup.py b/setup.py index eb32815..c711176 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ import setuptools.command.test import sys pkgdir = {"": "python%s" % sys.version_info[0]} -VERSION = '0.17.3' +VERSION = '0.17.4' # `python setup.py test` uses existing Python environment, no virtualenv, no pip. -- cgit v1.2.3 From a1457cc31f3206cf691d11d2bf34e98865873e9e Mon Sep 17 00:00:00 2001 From: Sergey Shepelev Date: Wed, 20 May 2020 14:56:12 +0300 Subject: IMPORTANT security vulnerability CWE-93 CRLF injection Force %xx quote of space, CR, LF characters in uri. Special thanks to Recar https://github.com/Ciyfly for discrete notification. https://cwe.mitre.org/data/definitions/93.html --- python2/httplib2/__init__.py | 3 +++ python3/httplib2/__init__.py | 3 +++ tests/__init__.py | 2 +- tests/test_http.py | 30 ++++++++++++++++++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/python2/httplib2/__init__.py b/python2/httplib2/__init__.py index 97e06c1..34281b7 100644 --- a/python2/httplib2/__init__.py +++ b/python2/httplib2/__init__.py @@ -1985,6 +1985,9 @@ class Http(object): headers["user-agent"] = "Python-httplib2/%s (gzip)" % __version__ uri = iri2uri(uri) + # Prevent CWE-75 space injection to manipulate request via part of uri. + # Prevent CWE-93 CRLF injection to modify headers via part of uri. + uri = uri.replace(" ", "%20").replace("\r", "%0D").replace("\n", "%0A") (scheme, authority, request_uri, defrag_uri) = urlnorm(uri) diff --git a/python3/httplib2/__init__.py b/python3/httplib2/__init__.py index 8785cc1..c0b1418 100644 --- a/python3/httplib2/__init__.py +++ b/python3/httplib2/__init__.py @@ -1790,6 +1790,9 @@ a string that contains the response entity body. headers["user-agent"] = "Python-httplib2/%s (gzip)" % __version__ uri = iri2uri(uri) + # Prevent CWE-75 space injection to manipulate request via part of uri. + # Prevent CWE-93 CRLF injection to modify headers via part of uri. + uri = uri.replace(" ", "%20").replace("\r", "%0D").replace("\n", "%0A") (scheme, authority, request_uri, defrag_uri) = urlnorm(uri) diff --git a/tests/__init__.py b/tests/__init__.py index a15db9e..02a3ecf 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -75,7 +75,7 @@ class BufferedReader(object): chunk = b"" else: chunk = self._sock.recv(8 << 10) - # print('!!! recv', chunk) + # print("!!! recv", chunk) if not chunk: self._end = True if untilend: diff --git a/tests/test_http.py b/tests/test_http.py index df99016..f61992c 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -703,3 +703,33 @@ def test_custom_redirect_codes(): response, content = http.request(uri, "GET") assert response.status == 301 assert response.previous is None + + +def test_cwe93_inject_crlf(): + # https://cwe.mitre.org/data/definitions/93.html + # GET /?q= HTTP/1.1 <- injected "HTTP/1.1" from attacker + # injected: attack + # ignore-http: HTTP/1.1 <- nominal "HTTP/1.1" from library + # Host: localhost:57285 + http = httplib2.Http() + with tests.server_reflect() as uri: + danger_url = urllib.parse.urljoin( + uri, "?q= HTTP/1.1\r\ninjected: attack\r\nignore-http:" + ) + response, content = http.request(danger_url, "GET") + assert response.status == 200 + req = tests.HttpRequest.from_bytes(content) + assert req.headers.get("injected") is None + + +def test_inject_space(): + # Injecting space into request line is precursor to CWE-93 and possibly other injections + http = httplib2.Http() + with tests.server_reflect() as uri: + # "\r\nignore-http:" suffix is nuance for current server implementation + # please only pay attention to space after "?q=" + danger_url = urllib.parse.urljoin(uri, "?q= HTTP/1.1\r\nignore-http:") + response, content = http.request(danger_url, "GET") + assert response.status == 200 + req = tests.HttpRequest.from_bytes(content) + assert req.uri == "/?q=%20HTTP/1.1%0D%0Aignore-http:" -- cgit v1.2.3 From 9fef207e85eef0534574d71fe1338c01874eba46 Mon Sep 17 00:00:00 2001 From: Sergey Shepelev Date: Wed, 20 May 2020 15:32:51 +0300 Subject: pyproject.toml package version in double quotes for black formatter --- pyproject.toml | 5 +++++ script/release | 18 +++++++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e8698a9 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,5 @@ +[build-system] +requires = ["setuptools", "wheel"] + +[tool.black] +line-length = 121 diff --git a/script/release b/script/release index 0f98e3e..2403275 100755 --- a/script/release +++ b/script/release @@ -45,9 +45,9 @@ auto_prepare_release() { last_tag=$(git tag --sort=-version:refname |head -n1) last_tag=${last_tag##v} version_replace="${last_tag}.post$(date -u +%y%m%d%H%M)" - update_version "setup.py" "s/VERSION =.+/VERSION = '$version_replace'/" - update_version "python2/httplib2/__init__.py" "s/__version__ =.+/__version__ = '$version_replace'/" - update_version "python3/httplib2/__init__.py" "s/__version__ =.+/__version__ = '$version_replace'/" + update_version "setup.py" "s/VERSION =.+/VERSION = \"$version_replace\"/" + update_version "python2/httplib2/__init__.py" "s/__version__ =.+/__version__ = \"$version_replace\"/" + update_version "python3/httplib2/__init__.py" "s/__version__ =.+/__version__ = \"$version_replace\"/" version_check "$version_replace" fi } @@ -132,9 +132,9 @@ bump_version() { fi echo "Next version: '$version_next'" >&2 - update_version "python3/httplib2/__init__.py" "s/__version__ =.+/__version__ = '$version_next'/" - update_version "python2/httplib2/__init__.py" "s/__version__ =.+/__version__ = '$version_next'/" - update_version "setup.py" "s/VERSION =.+/VERSION = '$version_next'/" + update_version "python3/httplib2/__init__.py" "s/__version__ =.+/__version__ = \"$version_next\"/" + update_version "python2/httplib2/__init__.py" "s/__version__ =.+/__version__ = \"$version_next\"/" + update_version "setup.py" "s/VERSION =.+/VERSION = \"$version_next\"/" confirm "Confirm changes? [yN] " || exit 1 } @@ -142,8 +142,8 @@ bump_version() { update_version() { local path="$1" local sed_expr="$2" - # sed -E --in-place='' -e "s/VERSION =.+/VERSION = '$version_replace'/" setup.py - # sed -E --in-place='' -e "s/__version__ =.+/__version__ = '$version_replace'/" python2/httplib2/__init__.py python3/httplib2/__init__.py + # sed -E --in-place='' -e "s/VERSION =.+/VERSION = \"$version_replace\"/" setup.py + # sed -E --in-place='' -e "s/__version__ =.+/__version__ = \"$version_replace\"/" python2/httplib2/__init__.py python3/httplib2/__init__.py echo "Updating file '$path'" >&2 if ! sed -E --in-place='' -e "$sed_expr" "$path" ; then echo "sed error $?" >&2 @@ -209,7 +209,7 @@ assert_tree_clean() { version_check() { local need=$1 - local version_setup=$(fgrep 'VERSION =' setup.py |tr -d " '" |cut -d\= -f2) + local version_setup=$(fgrep 'VERSION =' setup.py |tr -d " '\"" |cut -d\= -f2) local version_py2=$(cd python2 ; python2 -Es -c 'import httplib2;print(httplib2.__version__)') local version_py3=$(cd python3 ; python3 -Es -c 'import httplib2;print(httplib2.__version__)') if [[ "$version_setup" != "$need" ]] ; then -- cgit v1.2.3 From 8373177d3a9e4dd9c956f9bded22a5f96a00957b Mon Sep 17 00:00:00 2001 From: Sergey Shepelev Date: Wed, 20 May 2020 15:40:47 +0300 Subject: v0.18.0 release --- CHANGELOG | 7 +++++++ python2/httplib2/__init__.py | 2 +- python3/httplib2/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 4f4ca13..86c24a7 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,10 @@ +0.18.0 + + IMPORTANT security vulnerability CWE-93 CRLF injection + Force %xx quote of space, CR, LF characters in uri. + Special thanks to Recar https://github.com/Ciyfly for discrete notification. + https://cwe.mitre.org/data/definitions/93.html + 0.17.4 Ship test suite in source dist diff --git a/python2/httplib2/__init__.py b/python2/httplib2/__init__.py index 34281b7..d29193d 100644 --- a/python2/httplib2/__init__.py +++ b/python2/httplib2/__init__.py @@ -19,7 +19,7 @@ __contributors__ = [ "Alex Yu", ] __license__ = "MIT" -__version__ = '0.17.4' +__version__ = "0.18.0" import base64 import calendar diff --git a/python3/httplib2/__init__.py b/python3/httplib2/__init__.py index c0b1418..f5d7672 100644 --- a/python3/httplib2/__init__.py +++ b/python3/httplib2/__init__.py @@ -15,7 +15,7 @@ __contributors__ = [ "Alex Yu", ] __license__ = "MIT" -__version__ = '0.17.4' +__version__ = "0.18.0" import base64 import calendar diff --git a/setup.py b/setup.py index c711176..d8d9c48 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ import setuptools.command.test import sys pkgdir = {"": "python%s" % sys.version_info[0]} -VERSION = '0.17.4' +VERSION = "0.18.0" # `python setup.py test` uses existing Python environment, no virtualenv, no pip. -- cgit v1.2.3 From 828c26d8ca1e7e3c9c3e154885c9bf3a13426cbe Mon Sep 17 00:00:00 2001 From: Sergey Shepelev Date: Wed, 20 May 2020 20:44:00 +0300 Subject: Security Policy --- SECURITY.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..5eb3903 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,19 @@ +# Security Policy + +## Supported Versions + +master branch and latest release get priority support. You should expect all known problems fixed in master. + +All other released versions receive security updates per request. +If you use some old version and can not upgrade for any or no reason, ask for security update release, most likely you will get it. + +## Reporting a Vulnerability + +Contact current maintainers. At 2020-05: temotor@gmail.com or https://t.me/temotor +If that doesn't work, open Github issue just asking for private communication channel. + +This is volunteer maintained project, all issues are processed on best effort basis, no deadlines promised. Of course, security vulnerabilities get priority over regular issues. + +You can expect fame in history or maybe you prefer anonymity - say what you prefer. + +Thank you for responsible handling of security problems. Your attention and effort are appreciated. -- cgit v1.2.3 From 94f48efe2ffb1caa3fbcba0598e7583df02b832a Mon Sep 17 00:00:00 2001 From: Sergey Shepelev Date: Wed, 20 May 2020 22:42:35 +0300 Subject: check-manifest build tool --- .travis.yml | 2 +- MANIFEST.in | 5 +++++ pyproject.toml | 3 +++ script/release | 3 ++- 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 395c507..ed1aa72 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ cache: env: global: - - pip_install_common='pip>=9.0 setuptools>=36.2 wheel>=0.30' + - pip_install_common='pip>=9.0 setuptools>=43.0 wheel>=0.30' python: - 2.7 - 3.5 diff --git a/MANIFEST.in b/MANIFEST.in index 52ce781..412def6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,5 +2,10 @@ recursive-include python2 *.py *.txt recursive-include python3 *.py *.txt graft test graft tests +include *.md +include CHANGELOG +include LICENSE include python2/httplib2/test/*.txt include requirements*.txt +global-exclude __pycache__ +global-exclude *.py[cod] diff --git a/pyproject.toml b/pyproject.toml index e8698a9..fa137a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,3 +3,6 @@ requires = ["setuptools", "wheel"] [tool.black] line-length = 121 + +[tool.check-manifest] +ignore = [".travis.yml", "script/*", "*.tex"] diff --git a/script/release b/script/release index 2403275..a2ff80d 100755 --- a/script/release +++ b/script/release @@ -90,10 +90,11 @@ interactive() { local venv=./venv-release if [[ ! -d "$venv" ]] ; then virtualenv $venv - $venv/bin/pip install -U pip setuptools wheel twine + $venv/bin/pip install -U check-manifest pip 'setuptools>=43.0' wheel twine fi $venv/bin/python setup.py clean --all $venv/bin/python setup.py sdist bdist_wheel + $venv/bin/check-manifest || echo "FIXME check-manifest" >&2 if confirm "Upload to PyPI? Use in special situation, normally CI (Travis) will upload to PyPI. [yN] " ; then $venv/bin/twine upload dist/* || exit 1 -- cgit v1.2.3 From cb2940a5046c34b6c3568054e8679ae064da4f72 Mon Sep 17 00:00:00 2001 From: Sergey Shepelev Date: Wed, 20 May 2020 22:42:42 +0300 Subject: explicit build-backend workaround pip build isolation bug 6264 pip issue https://github.com/pypa/pip/issues/6264 workaround https://github.com/pypa/setuptools/issues/1694#issuecomment-466010982 fixes https://github.com/httplib2/httplib2/issues/169 --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fa137a3..5f7cbbd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,6 @@ [build-system] -requires = ["setuptools", "wheel"] +requires = ["setuptools >= 40.8.0", "wheel"] +build-backend = "setuptools.build_meta" [tool.black] line-length = 121 -- cgit v1.2.3 From 9bf300cdc372938f4237150d5b9b615879eb51a1 Mon Sep 17 00:00:00 2001 From: Sergey Shepelev Date: Wed, 20 May 2020 22:56:40 +0300 Subject: v0.18.1 release --- CHANGELOG | 6 ++++++ python2/httplib2/__init__.py | 2 +- python3/httplib2/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 86c24a7..2db1cc0 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,9 @@ +0.18.1 + + explicit build-backend workaround for pip build isolation bug + "AttributeError: 'module' object has no attribute '__legacy__'" on pip install + https://github.com/httplib2/httplib2/issues/169 + 0.18.0 IMPORTANT security vulnerability CWE-93 CRLF injection diff --git a/python2/httplib2/__init__.py b/python2/httplib2/__init__.py index d29193d..f35ba48 100644 --- a/python2/httplib2/__init__.py +++ b/python2/httplib2/__init__.py @@ -19,7 +19,7 @@ __contributors__ = [ "Alex Yu", ] __license__ = "MIT" -__version__ = "0.18.0" +__version__ = "0.18.1" import base64 import calendar diff --git a/python3/httplib2/__init__.py b/python3/httplib2/__init__.py index f5d7672..cf2db60 100644 --- a/python3/httplib2/__init__.py +++ b/python3/httplib2/__init__.py @@ -15,7 +15,7 @@ __contributors__ = [ "Alex Yu", ] __license__ = "MIT" -__version__ = "0.18.0" +__version__ = "0.18.1" import base64 import calendar diff --git a/setup.py b/setup.py index d8d9c48..b66d24e 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ import setuptools.command.test import sys pkgdir = {"": "python%s" % sys.version_info[0]} -VERSION = "0.18.0" +VERSION = "0.18.1" # `python setup.py test` uses existing Python environment, no virtualenv, no pip. -- cgit v1.2.3