[pull] master from sqlmapproject:master

data/txt/sha256sums.txt

@@ -170,7 +170,7 @@ ccec2373f6393f3d644db3de2910e17ef705817063c03e7ca4417f9d7f622527  lib/controller
 1da4ec9cd9b67c8b54e4a3d314f8237d58778d8f3a00bc26a1e0540294dca30f  lib/core/bigarray.py
 ed02b196398b8351ed6989c8fd8ec2a8244f2f9da6ca7b08691219dcc63422d8  lib/core/common.py
 a6397b10de7ae7c56ed6b0fa3b3c58eb7a9dbede61bf93d786e73258175c981e  lib/core/compat.py
-d6e80cecc32601e903aaf5faeb6fd2fe4c6b64a206d7eabb353b7a36e9f2bc46  lib/core/convert.py
+a9997e97ebe88e0bf7efcf21e878bc5f62c72348e5aba18f64d6861390a4dcf2  lib/core/convert.py
 c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6  lib/core/data.py
 421509c42dab738d908f2453cbdd6eb75eb672a7b6de68bee8c95d867fac79f1  lib/core/datatype.py
 90070160f9e8f166f9ea69975436fb358eaced6fec8a5947953b2cf050c51434  lib/core/decorators.py
@@ -182,21 +182,21 @@ f5272cda54f7cdd07fb6154d5a1ed1f1141a2a4f39b6a85d3f325fd60ac8dc9a  lib/core/enums
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/core/__init__.py
 914a13ee21fd610a6153a37cbe50830fcbd1324c7ebc1e7fc206d5e598b0f7ad  lib/core/log.py
 02a2264324caa249154e024a01bcd7cc40dbca4d647d5d10a50654b4415a6d77  lib/core/optiondict.py
-a9ead7442c8e1f34f03ad4db1145c08ee5907904c97e7dfd3202c752618b1092  lib/core/option.py
-fb0a08ac6f8bb07711e4e895eebf9fb3c8d452cc7aaebcdf78d926cdf051550d  lib/core/patch.py
+6576d40a66fa7871d3498c193f4e1e50a9fa9a380005d019c5c2266c1dc31c21  lib/core/option.py
+8171f6ee33e7742f06bb3014a28324496374beddee7b378ace10a26414a97762  lib/core/patch.py
 49c0fa7e3814dfda610d665ee02b12df299b28bc0b6773815b4395514ddf8dec  lib/core/profiling.py
 03db48f02c3d07a047ddb8fe33a757b6238867352d8ddda2a83e4fec09a98d04  lib/core/readlineng.py
-73ef0895d728fe76bf9abda94d4b97951069532a088d603a064e793bb2ae45d9  lib/core/replication.py
+48797d6c34dd9bb8a53f7f3794c85f4288d82a9a1d6be7fcf317d388cb20d4b3  lib/core/replication.py
 3574639db4942d16a2dc0a2f04bb7c0913c40c3862b54d34c44075a760e0c194  lib/core/revision.py
 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c  lib/core/session.py
-36bede69423a6b14b2ad9aa6c68150aec76b53cc91db4ca13339b59e7771faac  lib/core/settings.py
+c76cfbf08641ce4646cbe39584aa306ba0e5376c05af629eb10cc42b06debab9  lib/core/settings.py
 cd5a66deee8963ba8e7e9af3dd36eb5e8127d4d68698811c29e789655f507f82  lib/core/shell.py
-00dc9e87db2c13d7eaf18edd503267430460d91baf76760350be545d4a387a9f  lib/core/subprocessng.py
+bcb5d8090d5e3e0ef2a586ba09ba80eef0c6d51feb0f611ed25299fbb254f725  lib/core/subprocessng.py
 d35650179816193164a5f177102f18379dfbe6bb6d40fbb67b78d907b41c8038  lib/core/target.py
 85b7d6a724536bfcadd317972d4baec291e3813d6773921ee31755046a950a9a  lib/core/testing.py
 cf4dca323645d623109a82277a8e8a63eb9abb3fff6c8a57095eb171c1ef91b3  lib/core/threads.py
 b9aacb840310173202f79c2ba125b0243003ee6b44c92eca50424f2bdfc83c02  lib/core/unescaper.py
-492126b1f4c5ec0a352c507907a6f2067ec3a459250ed1c5d75f6457ef14a01f  lib/core/update.py
+10719f5ca450610ad28242017b2d8a77354ca357ffa26948c5f62d20cac29a8b  lib/core/update.py
 9ed5a0aef84f55d42894a006ff3616e8ee388a55790b04d968c80d1470c6d3bc  lib/core/wordlist.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/__init__.py
 54bfd31ebded3ffa5848df1c644f196eb704116517c7a3d860b5d081e984d821  lib/parse/banner.py
@@ -209,18 +209,18 @@ c5b258be7485089fac9d9cd179960e774fbd85e62836dc67cce76cc028bb6aeb  lib/parse/hand
 4ca378496510a02c0184b45107889625dc7faf459073e83b3520c66674049af4  lib/parse/payloads.py
 80d26a30abe948faf817a14f746cc8b3e2341ea8286830cccaae253b8ac0cdff  lib/parse/sitemap.py
 1be3da334411657461421b8a26a0f2ff28e1af1e28f1e963c6c92768f9b0847c  lib/request/basicauthhandler.py
-a30f18e52463c7c483430201b194350b55a54855507b253af826992e7e5c8435  lib/request/basic.py
+7302c38b8b418530a988b0294d09f0a9a08b7b35bc488956fee491a6dd88b2d1  lib/request/basic.py
 bc61bc944b81a7670884f82231033a6ac703324b34b071c9834886a92e249d0e  lib/request/chunkedhandler.py
 2daf0ce19eacda64687f441c90ef8da51714c3e8947c993ba08fb4ecdc4f5287  lib/request/comparison.py
 626bb6f3316a906a4629c0feb8ecbbcf473fb59e5bc532603c35b6b8f63f1deb  lib/request/connect.py
 8e06682280fce062eef6174351bfebcb6040e19976acff9dc7b3699779783498  lib/request/direct.py
 9ef303e18311e204727dac71c0ed8b814ab6aa1185f2af0a9703b95e5b3ea6e8  lib/request/dns.py
-ea553def411d6e208fb831a219b0241397fada46aaad432fc3c34addf75a336e  lib/request/httpshandler.py
+f56fc33251bd6214e3a6316c8f843eb192b2996aa84bd4c3e98790fdcf6e8cf0  lib/request/httpshandler.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/request/__init__.py
 aeeeb5f0148078e30d52208184042efc3618d3f2e840d7221897aae34315824e  lib/request/inject.py
 ada4d305d6ce441f79e52ec3f2fc23869ee2fa87c017723e8f3ed0dfa61cdab4  lib/request/methodrequest.py
-5c3edfca5ad58153ad6cface03777e059d3308b2aa3c38db993b5054145faa8e  lib/request/pkihandler.py
-4efead49b76d1237c283ecf281673d8762e09575d05af2a1e24680900ca83d0b  lib/request/rangehandler.py
+43a7fdf64e7ba63c6b2d641c9f999a63c12ac23b43b64fedfce4e05b863de568  lib/request/pkihandler.py
+b90feeb16e89a844427df42373b0139eb6f6cf3c48ccec32b3e3a3f540c2451e  lib/request/rangehandler.py
 47a97b264fb588142b102d18100030ce333ce372c677b97ed6cb04105c6c9d30  lib/request/redirecthandler.py
 1bf93c2c251f9c422ecf52d9cae0cd0ff4ea2e24091ee6d019c7a4f69de8e5eb  lib/request/templates.py
 01600295b17c00d4a5ada4c77aa688cfe36c89934da04c031be7da8040a3b457  lib/takeover/abstraction.py

lib/core/convert.py

@@ -295,7 +295,11 @@ def getBytes(value, encoding=None, errors="strict", unsafe=True):
     except (LookupError, TypeError):
         encoding = UNICODE_ENCODING
 
-    if isinstance(value, six.text_type):
+    if isinstance(value, bytearray):
+        return bytes(value)
+    elif isinstance(value, memoryview):
+        return value.tobytes()
+    elif isinstance(value, six.text_type):
         if INVALID_UNICODE_PRIVATE_AREA:
             if unsafe:
                 for char in xrange(0xF0000, 0xF00FF + 1):

lib/core/option.py

@@ -2038,7 +2038,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.cache.addrinfo = {}
     kb.cache.content = LRUDict(capacity=16)
     kb.cache.comparison = {}
-    kb.cache.encoding = {}
+    kb.cache.encoding = LRUDict(capacity=256)
     kb.cache.alphaBoundaries = None
     kb.cache.hashRegex = None
     kb.cache.intBoundaries = None

lib/core/patch.py

@@ -101,7 +101,7 @@ def _(self, *args):
 
     # Reference: https://github.com/sqlmapproject/sqlmap/issues/5929
     try:
-        global collections
+        import collections
         if not hasattr(collections, "MutableSet"):
             import collections.abc
             collections.MutableSet = collections.abc.MutableSet
@@ -139,7 +139,7 @@ def getargspec(func):
     # Installing "reversible" unicode (decoding) error handler
     def _reversible(ex):
         if INVALID_UNICODE_PRIVATE_AREA:
-            return (u"".join(_unichr(int('000f00%2x' % (_ if isinstance(_, int) else ord(_)), 16)) for _ in ex.object[ex.start:ex.end]), ex.end)
+            return (u"".join(_unichr(int('000f00%02x' % (_ if isinstance(_, int) else ord(_)), 16)) for _ in ex.object[ex.start:ex.end]), ex.end)
         else:
             return (u"".join(INVALID_UNICODE_CHAR_FORMAT % (_ if isinstance(_, int) else ord(_)) for _ in ex.object[ex.start:ex.end]), ex.end)
 

lib/core/replication.py

@@ -106,10 +106,12 @@ def select(self, condition=None):
             """
             This function is used for selecting row(s) from current table.
             """
-            _ = 'SELECT * FROM %s' % self.name
+            query = 'SELECT * FROM "%s"' % self.name
             if condition:
-                _ += 'WHERE %s' % condition
-            return self.execute(_)
+                query += ' WHERE %s' % condition
+
+            self.execute(query)
+            return self.parent.cursor.fetchall()
 
     def createTable(self, tblname, columns=None, typeless=False):
         """

lib/core/settings.py

@@ -19,7 +19,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.1.2"
+VERSION = "1.10.1.11"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/core/subprocessng.py

@@ -75,7 +75,7 @@ def recv(self, maxsize=None):
     def recv_err(self, maxsize=None):
         return self._recv('stderr', maxsize)
 
-    def send_recv(self, input='', maxsize=None):
+    def send_recv(self, input=b'', maxsize=None):
         return self.send(input), self.recv(maxsize), self.recv_err(maxsize)
 
     def get_conn_maxsize(self, which, maxsize):
@@ -97,7 +97,7 @@ def send(self, input):
             try:
                 x = msvcrt.get_osfhandle(self.stdin.fileno())
                 (_, written) = WriteFile(x, input)
-            except ValueError:
+            except (ValueError, NameError):
                 return self._close('stdin')
             except Exception as ex:
                 if getattr(ex, "args", None) and ex.args[0] in (109, errno.ESHUTDOWN):
@@ -187,7 +187,7 @@ def recv_some(p, t=.1, e=1, tr=5, stderr=0):
             y.append(r)
         else:
             time.sleep(max((x - time.time()) / tr, 0))
-    return b''.join(y)
+    return b''.join(getBytes(i) for i in y)
 
 def send_all(p, data):
     if not data:

lib/core/update.py

@@ -163,7 +163,7 @@ def update():
             infoMsg += "to use a GitHub for Windows client for updating "
             infoMsg += "purposes (https://desktop.github.com/) or just "
             infoMsg += "download the latest snapshot from "
-            infoMsg += "https://github.com/sqlmapproject/sqlmap/downloads"
+            infoMsg += "https://github.com/sqlmapproject/sqlmap/releases"
         else:
             infoMsg = "for Linux platform it's recommended "
             infoMsg += "to install a standard 'git' package (e.g.: 'apt install git')"

lib/request/basic.py

@@ -10,7 +10,6 @@
 import io
 import logging
 import re
-import struct
 import zlib
 
 from lib.core.common import Backend
@@ -259,7 +258,7 @@ def getHeuristicCharEncoding(page):
     'ascii'
     """
 
-    key = hash(page)
+    key = (len(page), hash(page))
     retVal = kb.cache.encoding[key] if key in kb.cache.encoding else detect(page[:HEURISTIC_PAGE_SIZE_THRESHOLD])["encoding"]
     kb.cache.encoding[key] = retVal
 
@@ -282,23 +281,25 @@ def decodePage(page, contentEncoding, contentType, percentDecode=True):
     if not page or (conf.nullConnection and len(page) < 2):
         return getUnicode(page)
 
-    contentEncoding = contentEncoding.lower() if hasattr(contentEncoding, "lower") else ""
-    contentType = contentType.lower() if hasattr(contentType, "lower") else ""
+    contentEncoding = getText(contentEncoding).lower() if contentEncoding else ""
+    contentType = getText(contentType).lower() if contentType else ""
 
     if contentEncoding in ("gzip", "x-gzip", "deflate"):
         if not kb.pageCompress:
             return None
 
         try:
             if contentEncoding == "deflate":
-                data = io.BytesIO(zlib.decompress(page, -15))  # Reference: http://stackoverflow.com/questions/1089662/python-inflate-and-deflate-implementations
+                obj = zlib.decompressobj(-15)
+                page = obj.decompress(page, MAX_CONNECTION_TOTAL_SIZE + 1)
+                page += obj.flush()
+                if len(page) > MAX_CONNECTION_TOTAL_SIZE:
+                    raise Exception("size too large")
             else:
                 data = gzip.GzipFile("", "rb", 9, io.BytesIO(page))
-                size = struct.unpack("<l", page[-4:])[0]  # Reference: http://pydoc.org/get.cgi/usr/local/lib/python2.5/gzip.py
-                if size > MAX_CONNECTION_TOTAL_SIZE:
+                page = data.read(MAX_CONNECTION_TOTAL_SIZE + 1)
+                if len(page) > MAX_CONNECTION_TOTAL_SIZE:
                     raise Exception("size too large")
-
-            page = data.read()
         except Exception as ex:
             if b"<html" not in page:  # in some cases, invalid "Content-Encoding" appears for plain HTML (should be ignored)
                 errMsg = "detected invalid data for declared content "

lib/request/httpshandler.py

@@ -65,6 +65,7 @@ def create_sock():
         #               https://www.mnot.net/blog/2014/12/27/python_2_and_tls_sni
         if hasattr(ssl, "SSLContext"):
             for protocol in (_ for _ in _protocols if _ >= ssl.PROTOCOL_TLSv1):
+                sock = None
                 try:
                     sock = create_sock()
                     if protocol not in _contexts:
@@ -94,6 +95,8 @@ def create_sock():
                         sock.close()
                 except (ssl.SSLError, socket.error, _http_client.BadStatusLine, AttributeError) as ex:
                     self._tunnel_host = None
+                    if sock:
+                        sock.close()
                     logger.debug("SSL connection error occurred for '%s' ('%s')" % (_lut[protocol], getSafeExString(ex)))
 
         elif hasattr(ssl, "wrap_socket"):

lib/request/pkihandler.py

@@ -5,12 +5,20 @@
 See the file 'LICENSE' for copying permission
 """
 
+ssl = None
+try:
+    import ssl as _ssl
+    ssl = _ssl
+except ImportError:
+    pass
+
 from lib.core.data import conf
 from lib.core.common import getSafeExString
 from lib.core.exception import SqlmapConnectionException
 from thirdparty.six.moves import http_client as _http_client
 from thirdparty.six.moves import urllib as _urllib
 
+
 class HTTPSPKIAuthHandler(_urllib.request.HTTPSHandler):
     def __init__(self, auth_file):
         _urllib.request.HTTPSHandler.__init__(self)
@@ -20,10 +28,24 @@ def https_open(self, req):
         return self.do_open(self.getConnection, req)
 
     def getConnection(self, host, timeout=None):
+        if timeout is None:
+            timeout = conf.timeout
+
+        if not hasattr(_http_client, "HTTPSConnection"):
+            raise SqlmapConnectionException("HTTPS support is not available in this Python build")
+
         try:
-            # Reference: https://docs.python.org/2/library/ssl.html#ssl.SSLContext.load_cert_chain
-            return _http_client.HTTPSConnection(host, cert_file=self.auth_file, key_file=self.auth_file, timeout=conf.timeout)
-        except IOError as ex:
+            if ssl and hasattr(ssl, "SSLContext") and hasattr(ssl, "create_default_context"):
+                ctx = ssl.create_default_context()
+                ctx.load_cert_chain(certfile=self.auth_file, keyfile=self.auth_file)
+                try:
+                    return _http_client.HTTPSConnection(host, timeout=timeout, context=ctx)
+                except TypeError:
+                    pass
+
+            return _http_client.HTTPSConnection(host, cert_file=self.auth_file, key_file=self.auth_file, timeout=timeout)
+
+        except (IOError, OSError) as ex:
             errMsg = "error occurred while using key "
             errMsg += "file '%s' ('%s')" % (self.auth_file, getSafeExString(ex))
             raise SqlmapConnectionException(errMsg)

lib/request/rangehandler.py

@@ -25,5 +25,5 @@ def http_error_206(self, req, fp, code, msg, hdrs):
     def http_error_416(self, req, fp, code, msg, hdrs):
         # HTTP's Range Not Satisfiable error
         errMsg = "there was a problem while connecting "
-        errMsg += "target ('406 - Range Not Satisfiable')"
+        errMsg += "target ('416 - Range Not Satisfiable')"
         raise SqlmapConnectionException(errMsg)