Merge 12d868f8d9 into 22210ca352

Fixes #628

.travis.yml

@@ -1,4 +1,6 @@
 language: python
+sudo: required
+dist: xenial
 
 addons:
   apt:
@@ -33,8 +35,13 @@ matrix:
       env: TOXENV=py35
     - python: 3.6
       env: TOXENV=py36
+    - python: 3.7
+      env: TOXENV=py37
+    - python: 3.8-dev
+      env: TOXENV=py38
     - python: pypy
       env: TOXENV=pypy
+      dist: trusty
 
 before_install:
   - if [[ $(echo "$TOXENV" | egrep -c "py35") != 0 ]]; then pyenv global system 3.5; fi;
@@ -42,7 +49,10 @@ before_install:
 install:
   - if [[ $(echo "$TOXENV" | egrep -c "py32") != 0 ]]; then pip install setuptools==17.1.1; fi;
   - if [[ $(echo "$TOXENV" | egrep -c "(py2[45]|py3[12])") != 0 ]]; then pip install virtualenv==1.7.2 tox==1.3; fi;
-  - if [[ $(echo "$TOXENV" | egrep -c "(py2[45]|py3[12])") == 0 ]]; then pip install tox; fi;
+  - if [[ $(echo "$TOXENV" | egrep -c "(py26|py33)") != 0 ]]; then pip install virtualenv==15.2.0 tox==2.9.1; fi;
+  - if [[ $(echo "$TOXENV" | egrep -c "(py2[456]|py3[123])") == 0 ]]; then pip install tox; fi;
+
+
 
 script:
   - tox

README.rst

@@ -7,9 +7,9 @@ speedtest.net
 .. image:: https://img.shields.io/pypi/v/speedtest-cli.svg
         :target: https://pypi.python.org/pypi/speedtest-cli/
         :alt: Latest Version
-.. image:: https://img.shields.io/pypi/dm/speedtest-cli.svg
+.. image:: https://img.shields.io/travis/sivel/speedtest-cli.svg
         :target: https://pypi.python.org/pypi/speedtest-cli/
-        :alt: Downloads
+        :alt: Travis
 .. image:: https://img.shields.io/pypi/l/speedtest-cli.svg
         :target: https://pypi.python.org/pypi/speedtest-cli/
         :alt: License
@@ -17,7 +17,7 @@ speedtest.net
 Versions
 --------
 
-speedtest-cli works with Python 2.4-3.6
+speedtest-cli works with Python 2.4-3.7
 
 .. image:: https://img.shields.io/pypi/pyversions/speedtest-cli.svg
         :target: https://pypi.python.org/pypi/speedtest-cli/
@@ -51,7 +51,8 @@ or
 ::
 
     git clone https://github.com/sivel/speedtest-cli.git
-    python speedtest-cli/setup.py install
+    cd speedtest-cli
+    python setup.py install
 
 Just download (Like the way it used to be)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -74,21 +75,23 @@ Usage
 ::
 
     $ speedtest-cli -h
-    usage: speedtest-cli [-h] [--no-download] [--no-upload] [--bytes] [--share]
-                         [--simple] [--csv] [--csv-delimiter CSV_DELIMITER]
-                         [--csv-header] [--json] [--list] [--server SERVER]
-                         [--exclude EXCLUDE] [--mini MINI] [--source SOURCE]
-                         [--timeout TIMEOUT] [--secure] [--no-pre-allocate]
-                         [--version]
-    
+    usage: speedtest-cli [-h] [--no-download] [--no-upload] [--single] [--bytes]
+                         [--share] [--simple] [--csv]
+                         [--csv-delimiter CSV_DELIMITER] [--csv-header] [--json]
+                         [--list] [--server SERVER] [--exclude EXCLUDE]
+                         [--mini MINI] [--source SOURCE] [--timeout TIMEOUT]
+                         [--secure] [--no-pre-allocate] [--version]
+
     Command line interface for testing internet bandwidth using speedtest.net.
     --------------------------------------------------------------------------
     https://github.com/sivel/speedtest-cli
-    
+
     optional arguments:
       -h, --help            show this help message and exit
       --no-download         Do not perform download test
       --no-upload           Do not perform upload test
+      --single              Only use a single connection instead of multiple. This
+                            simulates a typical file transfer.
       --bytes               Display values in bytes instead of bits. Does not
                             affect the image generated by --share, nor output from
                             --json or --csv

setup.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright 2012-2016 Matt Martz
+# Copyright 2012 Matt Martz
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -66,7 +66,7 @@ setup(
     author_email='matt@sivel.net',
     url='https://github.com/sivel/speedtest-cli',
     license='Apache License, Version 2.0',
-    py_modules=['speedtest', 'speedtest_cli'],
+    py_modules=['speedtest'],
     entry_points={
         'console_scripts': [
             'speedtest=speedtest:main',
@@ -91,5 +91,9 @@ setup(
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
     ]
 )

speedtest-cli.1

@@ -1,4 +1,4 @@
-.TH "speedtest-cli" 1 "2014-04-23" "speedtest-cli"
+.TH "speedtest-cli" 1 "2018-01-05" "speedtest-cli"
 .SH NAME
 speedtest\-cli \- Command line interface for testing internet bandwidth using speedtest.net
 .SH SYNOPSIS
@@ -23,14 +23,29 @@ Displays usage for the tool.
 
 .B Options
 
+\fB\-\-no\-download\fR
+.RS
+Do not perform download test
+.RE
+
+\fB\-\-no\-upload\fR
+.RS
+Do not perform upload test
+.RE
+
+\fB\-\-single\fR
+.RS
+Only use a single connection instead of multiple. This simulates a typical file transfer.
+.RE
+
 \fB\-\-bytes\fR
 .RS
-Display values in bytes instead of bits. Does not affect the image generated by \-\-share
+Display values in bytes instead of bits. Does not affect the image generated by \-\-share, nor output from \-\-json or \-\-csv
 .RE
 
 \fB\-\-share\fR
 .RS
-Generate and provide a URL to the speedtest.net share results image
+Generate and provide a URL to the speedtest.net share results image, not displayed with \-\-csv
 .RE
 
 \fB\-\-simple\fR
@@ -43,12 +58,12 @@ Suppress verbose output, only show basic information
 Suppress verbose output, only show basic information in CSV format. Speeds listed in bit/s and not affected by \-\-bytes
 .RE
 
-\fB\-\-csv-delimiter CSV_DELIMITER\fR
+\fB\-\-csv\-delimiter CSV_DELIMITER\fR
 .RS
 Single character delimiter to use in CSV output. Default ","
 .RE
 
-\fB\-\-csv-header\fR
+\fB\-\-csv\-header\fR
 .RS
 Print CSV headers
 .RE
@@ -65,7 +80,12 @@ Display a list of speedtest.net servers sorted by distance
 
 \fB\-\-server SERVER\fR
 .RS
-Specify a server ID to test against
+Specify a server ID to test against. Can be supplied multiple times
+.RE
+
+\fB\-\-exclude EXCLUDE\fR
+.RS
+Exclude a server from selection. Can be supplied multiple times
 .RE
 
 \fB\-\-mini MINI\fR
@@ -88,6 +108,11 @@ HTTP timeout in seconds. Default 10
 Use HTTPS instead of HTTP when communicating with speedtest.net operated servers
 .RE
 
+\fB\-\-no\-pre\-allocate\fR
+.RS
+Do not pre allocate upload data. Pre allocation is enabled by default to improve upload performance. To support systems with insufficient memory, use this option to avoid a MemoryError
+.RE
+
 \fB\-\-version\fR
 .RS
 Show the version number and exit

speedtest.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright 2012-2016 Matt Martz
+# Copyright 2012 Matt Martz
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -15,18 +15,18 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import os
-import re
 import csv
-import sys
-import math
+import datetime
 import errno
+import math
+import os
+import platform
+import re
 import signal
 import socket
-import timeit
-import datetime
-import platform
+import sys
 import threading
+import timeit
 import xml.parsers.expat
 
 try:
@@ -36,7 +36,7 @@ except ImportError:
     gzip = None
     GZIP_BASE = object
 
-__version__ = '2.0.0a'
+__version__ = '2.1.4b1'
 
 
 class FakeShutdownEvent(object):
@@ -49,10 +49,28 @@ class FakeShutdownEvent(object):
         "Dummy method to always return false"""
         return False
 
+    is_set = isSet
+
 
 # Some global variables we use
 DEBUG = False
 _GLOBAL_DEFAULT_TIMEOUT = object()
+PY25PLUS = sys.version_info[:2] >= (2, 5)
+PY26PLUS = sys.version_info[:2] >= (2, 6)
+PY32PLUS = sys.version_info[:2] >= (3, 2)
+PY310PLUS = sys.version_info[:2] >= (3, 10)
+SIZEMAP = {
+    "Mibyte": 8. * 1024 * 1024,
+    "Mbyte": 8. * 1000 * 1000,
+    "Mibit": 1. * 1024 * 1024,
+    "Mbit": 1. * 1000 * 1000,
+    "Kibyte": 8. * 1024,
+    "Kbyte": 8. * 1000,
+    "Kibit": 1. * 1024,
+    "Kbit": 1. * 1000,
+    "byte": 8.,
+    "bit": 1.,
+}
 
 # Begin import game to handle Python 2 and Python 3
 try:
@@ -64,13 +82,15 @@ except ImportError:
         json = None
 
 try:
-    import xml.etree.cElementTree as ET
-except ImportError:
+    import xml.etree.ElementTree as ET
     try:
-        import xml.etree.ElementTree as ET
+        from xml.etree.ElementTree import _Element as ET_Element
     except ImportError:
-        from xml.dom import minidom as DOM
-        ET = None
+        pass
+except ImportError:
+    from xml.dom import minidom as DOM
+    from xml.parsers.expat import ExpatError
+    ET = None
 
 try:
     from urllib2 import (urlopen, Request, HTTPError, URLError,
@@ -84,9 +104,9 @@ except ImportError:
                                 HTTPErrorProcessor, OpenerDirector)
 
 try:
-    from httplib import HTTPConnection
+    from httplib import HTTPConnection, BadStatusLine
 except ImportError:
-    from http.client import HTTPConnection
+    from http.client import HTTPConnection, BadStatusLine
 
 try:
     from httplib import HTTPSConnection
@@ -96,6 +116,11 @@ except ImportError:
     except ImportError:
         HTTPSConnection = None
 
+try:
+    from httplib import FakeSocket
+except ImportError:
+    FakeSocket = None
+
 try:
     from Queue import Queue
 except ImportError:
@@ -165,8 +190,14 @@ except ImportError:
             self.flush()
 
     _py3_print = getattr(builtins, 'print')
-    _py3_utf8_stdout = _Py3Utf8Output(sys.stdout)
-    _py3_utf8_stderr = _Py3Utf8Output(sys.stderr)
+    try:
+        _py3_utf8_stdout = _Py3Utf8Output(sys.stdout)
+        _py3_utf8_stderr = _Py3Utf8Output(sys.stderr)
+    except OSError:
+        # sys.stdout/sys.stderr is not a compatible stdout/stderr object
+        # just use it and hope things go ok
+        _py3_utf8_stdout = sys.stdout
+        _py3_utf8_stderr = sys.stderr
 
     def to_utf8(v):
         """No-op encode to utf-8 for py3"""
@@ -250,7 +281,6 @@ else:
             write(arg)
         write(end)
 
-
 # Exception "constants" to support Python 2 through Python 3
 try:
     import ssl
@@ -259,10 +289,30 @@ try:
     except AttributeError:
         CERT_ERROR = tuple()
 
-    HTTP_ERRORS = ((HTTPError, URLError, socket.error, ssl.SSLError) +
-                   CERT_ERROR)
+    HTTP_ERRORS = (
+        (HTTPError, URLError, socket.error, ssl.SSLError, BadStatusLine) +
+        CERT_ERROR
+    )
 except ImportError:
-    HTTP_ERRORS = (HTTPError, URLError, socket.error)
+    ssl = None
+    HTTP_ERRORS = (HTTPError, URLError, socket.error, BadStatusLine)
+
+if PY32PLUS:
+    etree_iter = ET.Element.iter
+elif PY25PLUS:
+    etree_iter = ET_Element.getiterator
+
+if PY26PLUS:
+    thread_is_alive = threading.Thread.is_alive
+else:
+    thread_is_alive = threading.Thread.isAlive
+
+
+def event_is_set(event):
+    try:
+        return event.is_set()
+    except AttributeError:
+        return event.isSet()
 
 
 class SpeedtestException(Exception):
@@ -278,7 +328,11 @@ class SpeedtestHTTPError(SpeedtestException):
 
 
 class SpeedtestConfigError(SpeedtestException):
-    """Configuration provided is invalid"""
+    """Configuration XML is invalid"""
+
+
+class SpeedtestServersError(SpeedtestException):
+    """Servers XML is invalid"""
 
 
 class ConfigRetrievalError(SpeedtestHTTPError):
@@ -378,13 +432,13 @@ class SpeedtestHTTPConnection(HTTPConnection):
     """
     def __init__(self, *args, **kwargs):
         source_address = kwargs.pop('source_address', None)
-        context = kwargs.pop('context', None)
         timeout = kwargs.pop('timeout', 10)
 
+        self._tunnel_host = None
+
         HTTPConnection.__init__(self, *args, **kwargs)
 
         self.source_address = source_address
-        self._context = context
         self.timeout = timeout
 
     def connect(self):
@@ -402,23 +456,75 @@ class SpeedtestHTTPConnection(HTTPConnection):
                 self.source_address
             )
 
+        if self._tunnel_host:
+            self._tunnel()
+
 
 if HTTPSConnection:
-    class SpeedtestHTTPSConnection(HTTPSConnection,
-                                   SpeedtestHTTPConnection):
+    class SpeedtestHTTPSConnection(HTTPSConnection):
         """Custom HTTPSConnection to support source_address across
         Python 2.4 - Python 3
         """
+        default_port = 443
+
+        def __init__(self, *args, **kwargs):
+            source_address = kwargs.pop('source_address', None)
+            timeout = kwargs.pop('timeout', 10)
+
+            self._tunnel_host = None
+
+            HTTPSConnection.__init__(self, *args, **kwargs)
+
+            self.timeout = timeout
+            self.source_address = source_address
+
         def connect(self):
             "Connect to a host on a given (SSL) port."
+            try:
+                self.sock = socket.create_connection(
+                    (self.host, self.port),
+                    self.timeout,
+                    self.source_address
+                )
+            except (AttributeError, TypeError):
+                self.sock = create_connection(
+                    (self.host, self.port),
+                    self.timeout,
+                    self.source_address
+                )
 
-            SpeedtestHTTPConnection.connect(self)
+            if self._tunnel_host:
+                self._tunnel()
 
-            kwargs = {}
-            if hasattr(ssl, 'SSLContext'):
-                kwargs['server_hostname'] = self.host
-
-            self.sock = self._context.wrap_socket(self.sock, **kwargs)
+            if ssl:
+                try:
+                    kwargs = {}
+                    if hasattr(ssl, 'SSLContext'):
+                        if self._tunnel_host:
+                            kwargs['server_hostname'] = self._tunnel_host
+                        else:
+                            kwargs['server_hostname'] = self.host
+                    self.sock = self._context.wrap_socket(self.sock, **kwargs)
+                except AttributeError:
+                    self.sock = ssl.wrap_socket(self.sock)
+                    try:
+                        self.sock.server_hostname = self.host
+                    except AttributeError:
+                        pass
+            elif FakeSocket:
+                # Python 2.4/2.5 support
+                try:
+                    self.sock = FakeSocket(self.sock, socket.ssl(self.sock))
+                except AttributeError:
+                    raise SpeedtestException(
+                        'This version of Python does not support HTTPS/SSL '
+                        'functionality'
+                    )
+            else:
+                raise SpeedtestException(
+                    'This version of Python does not support HTTPS/SSL '
+                    'functionality'
+                )
 
 
 def _build_connection(connection, source_address, timeout, context=None):
@@ -583,7 +689,8 @@ def build_user_agent():
 
     ua_tuple = (
         'Mozilla/5.0',
-        '(%s; U; %s; en-us)' % (platform.system(), platform.architecture()[0]),
+        '(%s; U; %s; en-us)' % (platform.platform(),
+                                platform.architecture()[0]),
         'Python/%s' % platform.python_version(),
         '(KHTML, like Gecko)',
         'speedtest-cli/%s' % __version__
@@ -642,6 +749,8 @@ def catch_request(request, opener=None):
 
     try:
         uh = _open(request)
+        if request.get_full_url() != uh.geturl():
+            printer('Redirected to %s' % uh.geturl(), debug=True)
         return uh, False
     except HTTP_ERRORS:
         e = get_exception()
@@ -681,7 +790,7 @@ def print_dots(shutdown_event):
     status
     """
     def inner(current, total, start=False, end=False):
-        if shutdown_event.isSet():
+        if event_is_set(shutdown_event):
             return
 
         sys.stdout.write('.')
@@ -720,7 +829,7 @@ class HTTPDownloader(threading.Thread):
         try:
             if (timeit.default_timer() - self.starttime) <= self.timeout:
                 f = self._opener(self.request)
-                while (not self._shutdown_event.isSet() and
+                while (not event_is_set(self._shutdown_event) and
                         (timeit.default_timer() - self.starttime) <=
                         self.timeout):
                     self.result.append(len(f.read(10240)))
@@ -729,6 +838,8 @@ class HTTPDownloader(threading.Thread):
                 f.close()
         except IOError:
             pass
+        except HTTP_ERRORS:
+            pass
 
 
 class HTTPUploaderData(object):
@@ -774,7 +885,7 @@ class HTTPUploaderData(object):
 
     def read(self, n=10240):
         if ((timeit.default_timer() - self.start) <= self.timeout and
-                not self._shutdown_event.isSet()):
+                not event_is_set(self._shutdown_event)):
             chunk = self.data.read(n)
             self.total.append(len(chunk))
             return chunk
@@ -794,7 +905,7 @@ class HTTPUploader(threading.Thread):
         self.request = request
         self.request.data.start = self.starttime = start
         self.size = size
-        self.result = None
+        self.result = 0
         self.timeout = timeout
         self.i = i
 
@@ -812,7 +923,7 @@ class HTTPUploader(threading.Thread):
         request = self.request
         try:
             if ((timeit.default_timer() - self.starttime) <= self.timeout and
-                    not self._shutdown_event.isSet()):
+                    not event_is_set(self._shutdown_event)):
                 try:
                     f = self._opener(request)
                 except TypeError:
@@ -829,6 +940,8 @@ class HTTPUploader(threading.Thread):
                 self.result = 0
         except (IOError, SpeedtestUploadTimeout):
             self.result = sum(self.request.data.total)
+        except HTTP_ERRORS:
+            self.result = 0
 
 
 class SpeedtestResults(object):
@@ -856,7 +969,7 @@ class SpeedtestResults(object):
         self.client = client or {}
 
         self._share = None
-        self.timestamp = '%sZ' % datetime.datetime.utcnow().isoformat()
+        self.timestamp = '%sZ' % datetime.datetime.now(datetime.UTC).isoformat()
         self.bytes_received = 0
         self.bytes_sent = 0
 
@@ -1017,10 +1130,7 @@ class Speedtest(object):
     @property
     def best(self):
         if not self._best:
-            raise SpeedtestMissingBestServer(
-                'get_best_server not called or not able to determine best '
-                'server'
-            )
+            self.get_best_server()
         return self._best
 
     def get_config(self):
@@ -1036,13 +1146,16 @@ class Speedtest(object):
         uh, e = catch_request(request, opener=self._opener)
         if e:
             raise ConfigRetrievalError(e)
-        configxml = []
+        configxml_list = []
 
         stream = get_response_stream(uh)
 
         while 1:
-            configxml.append(stream.read(1024))
-            if len(configxml[-1]) == 0:
+            try:
+                configxml_list.append(stream.read(1024))
+            except (OSError, EOFError):
+                raise ConfigRetrievalError(get_exception())
+            if len(configxml_list[-1]) == 0:
                 break
         stream.close()
         uh.close()
@@ -1050,10 +1163,18 @@ class Speedtest(object):
         if int(uh.code) != 200:
             return None
 
-        printer('Config XML:\n%s' % ''.encode().join(configxml), debug=True)
+        configxml = ''.encode().join(configxml_list)
+
+        printer('Config XML:\n%s' % configxml, debug=True)
 
         try:
-            root = ET.fromstring(''.encode().join(configxml))
+            try:
+                root = ET.fromstring(configxml)
+            except ET.ParseError:
+                e = get_exception()
+                raise SpeedtestConfigError(
+                    'Malformed speedtest.net configuration: %s' % e
+                )
             server_config = root.find('server-config').attrib
             download = root.find('download').attrib
             upload = root.find('upload').attrib
@@ -1061,16 +1182,22 @@ class Speedtest(object):
             client = root.find('client').attrib
 
         except AttributeError:
-            root = DOM.parseString(''.join(configxml))
+            try:
+                root = DOM.parseString(configxml)
+            except ExpatError:
+                e = get_exception()
+                raise SpeedtestConfigError(
+                    'Malformed speedtest.net configuration: %s' % e
+                )
             server_config = get_attributes_by_tag_name(root, 'server-config')
             download = get_attributes_by_tag_name(root, 'download')
             upload = get_attributes_by_tag_name(root, 'upload')
             # times = get_attributes_by_tag_name(root, 'times')
             client = get_attributes_by_tag_name(root, 'client')
 
-        ignore_servers = list(
-            map(int, server_config['ignoreids'].split(','))
-        )
+        ignore_servers = [
+            int(i) for i in server_config['ignoreids'].split(',') if i
+        ]
 
         ratio = int(upload['ratio'])
         upload_max = int(upload['maxchunkcount'])
@@ -1110,7 +1237,13 @@ class Speedtest(object):
             'upload_max': upload_count * size_count
         })
 
-        self.lat_lon = (float(client['lat']), float(client['lon']))
+        try:
+            self.lat_lon = (float(client['lat']), float(client['lon']))
+        except ValueError:
+            raise SpeedtestConfigError(
+                'Unknown location: lat=%r lon=%r' %
+                (client.get('lat'), client.get('lon'))
+            )
 
         printer('Config:\n%r' % self.config, debug=True)
 
@@ -1164,10 +1297,13 @@ class Speedtest(object):
 
                 stream = get_response_stream(uh)
 
-                serversxml = []
+                serversxml_list = []
                 while 1:
-                    serversxml.append(stream.read(1024))
-                    if len(serversxml[-1]) == 0:
+                    try:
+                        serversxml_list.append(stream.read(1024))
+                    except (OSError, EOFError):
+                        raise ServersRetrievalError(get_exception())
+                    if len(serversxml_list[-1]) == 0:
                         break
 
                 stream.close()
@@ -1176,15 +1312,28 @@ class Speedtest(object):
                 if int(uh.code) != 200:
                     raise ServersRetrievalError()
 
-                printer('Servers XML:\n%s' % ''.encode().join(serversxml),
-                        debug=True)
+                serversxml = ''.encode().join(serversxml_list)
+
+                printer('Servers XML:\n%s' % serversxml, debug=True)
 
                 try:
                     try:
-                        root = ET.fromstring(''.encode().join(serversxml))
-                        elements = root.getiterator('server')
+                        try:
+                            root = ET.fromstring(serversxml)
+                        except ET.ParseError:
+                            e = get_exception()
+                            raise SpeedtestServersError(
+                                'Malformed speedtest.net server list: %s' % e
+                            )
+                        elements = etree_iter(root, 'server')
                     except AttributeError:
-                        root = DOM.parseString(''.join(serversxml))
+                        try:
+                            root = DOM.parseString(serversxml)
+                        except ExpatError:
+                            e = get_exception()
+                            raise SpeedtestServersError(
+                                'Malformed speedtest.net server list: %s' % e
+                            )
                         elements = root.getElementsByTagName('server')
                 except (SyntaxError, xml.parsers.expat.ExpatError):
                     raise ServersRetrievalError()
@@ -1376,8 +1525,12 @@ class Speedtest(object):
         printer('Best Server:\n%r' % best, debug=True)
         return best
 
-    def download(self, callback=do_nothing):
-        """Test download speed against speedtest.net"""
+    def download(self, callback=do_nothing, threads=None):
+        """Test download speed against speedtest.net
+
+        A ``threads`` value of ``None`` will fall back to those dictated
+        by the speedtest.net configuration
+        """
 
         urls = []
         for size in self.config['sizes']['download']:
@@ -1392,6 +1545,9 @@ class Speedtest(object):
                 build_request(url, bump=i, secure=self._secure)
             )
 
+        max_threads = threads or self.config['threads']['download']
+        in_flight = {'threads': 0}
+
         def producer(q, requests, request_count):
             for i, request in enumerate(requests):
                 thread = HTTPDownloader(
@@ -1402,21 +1558,26 @@ class Speedtest(object):
                     opener=self._opener,
                     shutdown_event=self._shutdown_event
                 )
+                while in_flight['threads'] >= max_threads:
+                    timeit.time.sleep(0.001)
                 thread.start()
                 q.put(thread, True)
+                in_flight['threads'] += 1
                 callback(i, request_count, start=True)
 
         finished = []
 
         def consumer(q, request_count):
+            _is_alive = thread_is_alive
             while len(finished) < request_count:
                 thread = q.get(True)
-                while thread.isAlive():
-                    thread.join(timeout=0.1)
+                while _is_alive(thread):
+                    thread.join(timeout=0.001)
+                in_flight['threads'] -= 1
                 finished.append(sum(thread.result))
                 callback(thread.i, request_count, end=True)
 
-        q = Queue(self.config['threads']['download'])
+        q = Queue(max_threads)
         prod_thread = threading.Thread(target=producer,
                                        args=(q, requests, request_count))
         cons_thread = threading.Thread(target=consumer,
@@ -1424,10 +1585,11 @@ class Speedtest(object):
         start = timeit.default_timer()
         prod_thread.start()
         cons_thread.start()
-        while prod_thread.isAlive():
-            prod_thread.join(timeout=0.1)
-        while cons_thread.isAlive():
-            cons_thread.join(timeout=0.1)
+        _is_alive = thread_is_alive
+        while _is_alive(prod_thread):
+            prod_thread.join(timeout=0.001)
+        while _is_alive(cons_thread):
+            cons_thread.join(timeout=0.001)
 
         stop = timeit.default_timer()
         self.results.bytes_received = sum(finished)
@@ -1438,8 +1600,12 @@ class Speedtest(object):
             self.config['threads']['upload'] = 8
         return self.results.download
 
-    def upload(self, callback=do_nothing, pre_allocate=True):
-        """Test upload speed against speedtest.net"""
+    def upload(self, callback=do_nothing, pre_allocate=True, threads=None):
+        """Test upload speed against speedtest.net
+
+        A ``threads`` value of ``None`` will fall back to those dictated
+        by the speedtest.net configuration
+        """
 
         sizes = []
 
@@ -1462,13 +1628,19 @@ class Speedtest(object):
             )
             if pre_allocate:
                 data.pre_allocate()
+
+            headers = {'Content-length': size}
             requests.append(
                 (
-                    build_request(self.best['url'], data, secure=self._secure),
+                    build_request(self.best['url'], data, secure=self._secure,
+                                  headers=headers),
                     size
                 )
             )
 
+        max_threads = threads or self.config['threads']['upload']
+        in_flight = {'threads': 0}
+
         def producer(q, requests, request_count):
             for i, request in enumerate(requests[:request_count]):
                 thread = HTTPUploader(
@@ -1480,21 +1652,26 @@ class Speedtest(object):
                     opener=self._opener,
                     shutdown_event=self._shutdown_event
                 )
+                while in_flight['threads'] >= max_threads:
+                    timeit.time.sleep(0.001)
                 thread.start()
                 q.put(thread, True)
+                in_flight['threads'] += 1
                 callback(i, request_count, start=True)
 
         finished = []
 
         def consumer(q, request_count):
+            _is_alive = thread_is_alive
             while len(finished) < request_count:
                 thread = q.get(True)
-                while thread.isAlive():
-                    thread.join(timeout=0.1)
+                while _is_alive(thread):
+                    thread.join(timeout=0.001)
+                in_flight['threads'] -= 1
                 finished.append(thread.result)
                 callback(thread.i, request_count, end=True)
 
-        q = Queue(self.config['threads']['upload'])
+        q = Queue(threads or self.config['threads']['upload'])
         prod_thread = threading.Thread(target=producer,
                                        args=(q, requests, request_count))
         cons_thread = threading.Thread(target=consumer,
@@ -1502,9 +1679,10 @@ class Speedtest(object):
         start = timeit.default_timer()
         prod_thread.start()
         cons_thread.start()
-        while prod_thread.isAlive():
+        _is_alive = thread_is_alive
+        while _is_alive(prod_thread):
             prod_thread.join(timeout=0.1)
-        while cons_thread.isAlive():
+        while _is_alive(cons_thread):
             cons_thread.join(timeout=0.1)
 
         stop = timeit.default_timer()
@@ -1521,7 +1699,7 @@ def ctrl_c(shutdown_event):
     """
     def inner(signum, frame):
         shutdown_event.set()
-        print_('\nCancelling...')
+        printer('\nCancelling...', error=True)
         sys.exit(0)
     return inner
 
@@ -1529,14 +1707,15 @@ def ctrl_c(shutdown_event):
 def version():
     """Print the version"""
 
-    print_(__version__)
+    printer('speedtest-cli %s' % __version__)
+    printer('Python %s' % sys.version.replace('\n', ''))
     sys.exit(0)
 
 
 def csv_header(delimiter=','):
     """Print the CSV Headers"""
 
-    print_(SpeedtestResults.csv_header(delimiter=delimiter))
+    printer(SpeedtestResults.csv_header(delimiter=delimiter))
     sys.exit(0)
 
 
@@ -1562,11 +1741,18 @@ def parse_args():
     parser.add_argument('--no-upload', dest='upload', default=True,
                         action='store_const', const=False,
                         help='Do not perform upload test')
-    parser.add_argument('--bytes', dest='units', action='store_const',
-                        const=('byte', 8), default=('bit', 1),
+    parser.add_argument('--single', default=False, action='store_true',
+                        help='Only use a single connection instead of '
+                             'multiple. This simulates a typical file '
+                             'transfer.')
+    parser.add_argument('--bytes', default=False, action='store_true',
                         help='Display values in bytes instead of bits. Does '
                              'not affect the image generated by --share, nor '
                              'output from --json or --csv')
+    parser.add_argument('--units', choices=tuple(SIZEMAP),
+                        help='Determines which units to display values in. Does '
+                             'not affect the image generated by --share, nor '
+                             'output from --json or --csv')
     parser.add_argument('--share', action='store_true',
                         help='Generate and provide a URL to the speedtest.net '
                              'share results image, not displayed with --csv')
@@ -1576,7 +1762,7 @@ def parse_args():
     parser.add_argument('--csv', action='store_true', default=False,
                         help='Suppress verbose output, only show basic '
                              'information in CSV format. Speeds listed in '
-                             'bit/s and not affected by --bytes')
+                             'bit/s and not affected by --bytes or --units')
     parser.add_argument('--csv-delimiter', default=',', type=PARSER_TYPE_STR,
                         help='Single character delimiter to use in CSV '
                              'output. Default ","')
@@ -1585,7 +1771,7 @@ def parse_args():
     parser.add_argument('--json', action='store_true', default=False,
                         help='Suppress verbose output, only show basic '
                              'information in JSON format. Speeds listed in '
-                             'bit/s and not affected by --bytes')
+                             'bit/s and not affected by --bytes or --units')
     parser.add_argument('--list', action='store_true',
                         help='Display a list of speedtest.net servers '
                              'sorted by distance')
@@ -1641,13 +1827,16 @@ def validate_optional_args(args):
 
 
 def printer(string, quiet=False, debug=False, error=False, **kwargs):
-    """Helper function to print a string only when not quiet"""
+    """Helper function print a string with various features"""
 
     if debug and not DEBUG:
         return
 
     if debug:
-        out = '\033[1;30mDEBUG: %s\033[0m' % string
+        if sys.stdout.isatty():
+            out = '\033[1;30mDEBUG: %s\033[0m' % string
+        else:
+            out = 'DEBUG: %s' % string
     else:
         out = string
 
@@ -1676,6 +1865,10 @@ def shell():
         raise SpeedtestCLIError('Cannot supply both --no-download and '
                                 '--no-upload')
 
+    if args.bytes and args.units:
+        raise SpeedtestCLIError('Cannot supply both --bytes and '
+                                '--units')
+
     if len(args.csv_delimiter) != 1:
         raise SpeedtestCLIError('--csv-delimiter must be a single character')
 
@@ -1690,6 +1883,13 @@ def shell():
     if debug:
         DEBUG = True
 
+    if args.bytes:
+        units = 'Mbyte'
+    elif args.units:
+        units = args.units
+    else:
+        units = 'Mbit'
+
     if args.simple or args.csv or args.json:
         quiet = True
     else:
@@ -1729,7 +1929,7 @@ def shell():
                 line = ('%(id)5s) %(sponsor)s (%(name)s, %(country)s) '
                         '[%(d)0.2f km]' % server)
                 try:
-                    print_(line)
+                    printer(line)
                 except IOError:
                     e = get_exception()
                     if e.errno != errno.EPIPE:
@@ -1773,10 +1973,13 @@ def shell():
     if args.download:
         printer('Testing download speed', quiet,
                 end=('', '\n')[bool(debug)])
-        speedtest.download(callback=callback)
-        printer('Download: %0.2f M%s/s' %
-                ((results.download / 1000.0 / 1000.0) / args.units[1],
-                 args.units[0]),
+        speedtest.download(
+            callback=callback,
+            threads=(None, 1)[args.single]
+        )
+        printer('Download: %0.2f %s/s' %
+                (results.download / SIZEMAP[units],
+                 units),
                 quiet)
     else:
         printer('Skipping download test', quiet)
@@ -1784,29 +1987,34 @@ def shell():
     if args.upload:
         printer('Testing upload speed', quiet,
                 end=('', '\n')[bool(debug)])
-        speedtest.upload(callback=callback, pre_allocate=args.pre_allocate)
-        printer('Upload: %0.2f M%s/s' %
-                ((results.upload / 1000.0 / 1000.0) / args.units[1],
-                 args.units[0]),
+        speedtest.upload(
+            callback=callback,
+            pre_allocate=args.pre_allocate,
+            threads=(None, 1)[args.single]
+        )
+        printer('Upload: %0.2f %s/s' %
+                (results.upload / SIZEMAP[units],
+                 units),
                 quiet)
     else:
         printer('Skipping upload test', quiet)
 
     printer('Results:\n%r' % results.dict(), debug=True)
 
+    if not args.simple and args.share:
+        results.share()
+
     if args.simple:
-        print_('Ping: %s ms\nDownload: %0.2f M%s/s\nUpload: %0.2f M%s/s' %
-               (results.ping,
-                (results.download / 1000.0 / 1000.0) / args.units[1],
-                args.units[0],
-                (results.upload / 1000.0 / 1000.0) / args.units[1],
-                args.units[0]))
+        printer('Ping: %s ms\nDownload: %0.2f M%s/s\nUpload: %0.2f M%s/s' %
+                (results.ping,
+                 (results.download / 1000.0 / 1000.0) / args.units[1],
+                 args.units[0],
+                 (results.upload / 1000.0 / 1000.0) / args.units[1],
+                 args.units[0]))
     elif args.csv:
-        print_(results.csv(delimiter=args.csv_delimiter))
+        printer(results.csv(delimiter=args.csv_delimiter))
     elif args.json:
-        if args.share:
-            results.share()
-        print_(results.json())
+        printer(results.json())
 
     if args.share and not machine_format:
         printer('Share results: %s' % results.share())
@@ -1816,12 +2024,15 @@ def main():
     try:
         shell()
     except KeyboardInterrupt:
-        print_('\nCancelling...')
+        printer('\nCancelling...', error=True)
     except (SpeedtestException, SystemExit):
         e = get_exception()
         # Ignore a successful exit, or argparse exit
         if getattr(e, 'code', 1) not in (0, 2):
-            raise SystemExit('ERROR: %s' % e)
+            msg = '%s' % e
+            if not msg:
+                msg = '%r' % e
+            raise SystemExit('ERROR: %s' % msg)
 
 
 if __name__ == '__main__':

tests/scripts/source.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright 2012-2016 Matt Martz
+# Copyright 2018 Matt Martz
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -15,20 +15,23 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import warnings
+import sys
+import subprocess
 
-DEPRECATED_MSG = ('The file speedtest_cli.py has been deprecated in favor of '
-                  'speedtest.py\nand is available for download at:\n\n'
-                  'https://raw.githubusercontent.com/sivel/speedtest-cli/'
-                  'master/speedtest.py')
+cmd = [sys.executable, 'speedtest.py', '--source', '127.0.0.1']
 
+p = subprocess.Popen(
+    cmd,
+    stdout=subprocess.PIPE,
+    stderr=subprocess.PIPE
+)
 
-if __name__ == '__main__':
-    raise SystemExit(DEPRECATED_MSG)
-else:
-    try:
-        from speedtest import *
-    except ImportError:
-        raise SystemExit(DEPRECATED_MSG)
-    else:
-        warnings.warn(DEPRECATED_MSG, UserWarning)
+stdout, stderr = p.communicate()
+
+if p.returncode != 1:
+    raise SystemExit('%s did not fail with exit code 1' % ' '.join(cmd))
+
+if 'Invalid argument'.encode() not in stderr:
+    raise SystemExit(
+        '"Invalid argument" not found in stderr:\n%s' % stderr.decode()
+    )

tox.ini

@@ -6,6 +6,8 @@ commands =
     {envpython} -V
     {envpython} -m compileall speedtest.py
     {envpython} speedtest.py
+    {envpython} speedtest.py --source 172.17.0.1
+    {envpython} tests/scripts/source.py
 
 [testenv:flake8]
 basepython=python
@@ -19,3 +21,5 @@ commands =
     pypy -V
     pypy -m compileall speedtest.py
     pypy speedtest.py
+    pypy speedtest.py --source 172.17.0.1
+    pypy tests/scripts/source.py