v7.0.22

Use correct argument --exclude instead of --ignore

HISTORY.rst

@@ -3,6 +3,267 @@ History
 -------
 
 
+7.0.22 (2017-06-08)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v8.0.3.
+- Improve Matlab language detection.
+
+
+7.0.21 (2017-05-24)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v8.0.2.
+- Only treat proxy string as NTLM proxy after unable to connect with HTTPS and
+  SOCKS proxy.
+- Support running automated tests on Linux, OS X, and Windows.
+- Ability to disable SSL cert verification. wakatime/wakatime#90
+- Disable line count stats for files larger than 2MB to improve performance.
+- Print error saying Python needs upgrading when requests can't be imported.
+
+
+7.0.20 (2017-04-10)
+++++++++++++++++++
+
+- Fix install instructions formatting.
+
+
+7.0.19 (2017-04-10)
+++++++++++++++++++
+
+- Remove /var/www/ from default ignored folders.
+
+
+7.0.18 (2017-03-16)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v8.0.0.
+- No longer creating ~/.wakatime.cfg file, since only using Sublime settings.
+
+
+7.0.17 (2017-03-01)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v7.0.4.
+
+
+7.0.16 (2017-02-20)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v7.0.2.
+
+
+7.0.15 (2017-02-13)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.2.2.
+- Upgrade pygments library to v2.2.0 for improved language detection.
+
+
+7.0.14 (2017-02-08)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.2.1.
+- Allow boolean or list of regex patterns for hidefilenames config setting.
+
+
+7.0.13 (2016-11-11)
+++++++++++++++++++
+
+- Support old Sublime Text with Python 2.6.
+- Fix bug that prevented reading default api key from existing config file.
+
+
+7.0.12 (2016-10-24)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.2.0.
+- Exit with status code 104 when api key is missing or invalid. Exit with
+  status code 103 when config file missing or invalid.
+- New WAKATIME_HOME env variable for setting path to config and log files.
+- Improve debug warning message from unsupported dependency parsers.
+
+
+7.0.11 (2016-09-23)
+++++++++++++++++++
+
+- Handle UnicodeDecodeError when when logging. Related to #68.
+
+
+7.0.10 (2016-09-22)
+++++++++++++++++++
+
+- Handle UnicodeDecodeError when looking for python. Fixes #68.
+- Upgrade wakatime-cli to v6.0.9.
+
+
+7.0.9 (2016-09-02)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.0.8.
+
+
+7.0.8 (2016-07-21)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to master version to fix debug logging encoding bug.
+
+
+7.0.7 (2016-07-06)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.0.7.
+- Handle unknown exceptions from requests library by deleting cached session
+  object because it could be from a previous conflicting version.
+- New hostname setting in config file to set machine hostname. Hostname
+  argument takes priority over hostname from config file.
+- Prevent logging unrelated exception when logging tracebacks.
+- Use correct namespace for pygments.lexers.ClassNotFound exception so it is
+  caught when dependency detection not available for a language.
+
+
+7.0.6 (2016-06-13)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.0.5.
+- Upgrade pygments to v2.1.3 for better language coverage.
+
+
+7.0.5 (2016-06-08)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to master version to fix bug in urllib3 package causing
+  unhandled retry exceptions.
+- Prevent tracking git branch with detached head.
+
+
+7.0.4 (2016-05-21)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.0.3.
+- Upgrade requests dependency to v2.10.0.
+- Support for SOCKS proxies.
+
+
+7.0.3 (2016-05-16)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.0.2.
+- Prevent popup on Mac when xcode-tools is not installed.
+
+
+7.0.2 (2016-04-29)
+++++++++++++++++++
+
+- Prevent implicit unicode decoding from string format when logging output
+  from Python version check.
+
+
+7.0.1 (2016-04-28)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.0.1.
+- Fix bug which prevented plugin from being sent with extra heartbeats.
+
+
+7.0.0 (2016-04-28)
+++++++++++++++++++
+
+- Queue heartbeats and send to wakatime-cli after 4 seconds.
+- Nest settings menu under Package Settings.
+- Upgrade wakatime-cli to v6.0.0.
+- Increase default network timeout to 60 seconds when sending heartbeats to
+  the api.
+- New --extra-heartbeats command line argument for sending a JSON array of
+  extra queued heartbeats to STDIN.
+- Change --entitytype command line argument to --entity-type.
+- No longer allowing --entity-type of url.
+- Support passing an alternate language to cli to be used when a language can
+  not be guessed from the code file.
+
+
+6.0.8 (2016-04-18)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v5.0.0.
+- Support regex patterns in projectmap config section for renaming projects.
+- Upgrade pytz to v2016.3.
+- Upgrade tzlocal to v1.2.2.
+
+
+6.0.7 (2016-03-11)
+++++++++++++++++++
+
+- Fix bug causing RuntimeError when finding Python location
+
+
+6.0.6 (2016-03-06)
+++++++++++++++++++
+
+- upgrade wakatime-cli to v4.1.13
+- encode TimeZone as utf-8 before adding to headers
+- encode X-Machine-Name as utf-8 before adding to headers
+
+
+6.0.5 (2016-03-06)
+++++++++++++++++++
+
+- upgrade wakatime-cli to v4.1.11
+- encode machine hostname as Unicode when adding to X-Machine-Name header
+
+
+6.0.4 (2016-01-15)
+++++++++++++++++++
+
+- fix UnicodeDecodeError on ST2 with non-English locale
+
+
+6.0.3 (2016-01-11)
+++++++++++++++++++
+
+- upgrade wakatime-cli core to v4.1.10
+- accept 201 or 202 response codes as success from api
+- upgrade requests package to v2.9.1
+
+
+6.0.2 (2016-01-06)
+++++++++++++++++++
+
+- upgrade wakatime-cli core to v4.1.9
+- improve C# dependency detection
+- correctly log exception tracebacks
+- log all unknown exceptions to wakatime.log file
+- disable urllib3 SSL warning from every request
+- detect dependencies from golang files
+- use api.wakatime.com for sending heartbeats
+
+
+6.0.1 (2016-01-01)
+++++++++++++++++++
+
+- use embedded python if system python is broken, or doesn't output a version number
+- log output from wakatime-cli in ST console when in debug mode
+
+
+6.0.0 (2015-12-01)
+++++++++++++++++++
+
+- use embeddable Python instead of installing on Windows
+
+
+5.0.1 (2015-10-06)
+++++++++++++++++++
+
+- look for python in system PATH again
+
+
+5.0.0 (2015-10-02)
+++++++++++++++++++
+
+- improve logging with levels and log function
+- switch registry warnings to debug log level
+
+
 4.0.20 (2015-10-01)
 ++++++++++++++++++
 

Main.sublime-menu

@@ -6,24 +6,37 @@
         "children":
         [
             {
-                "caption": "WakaTime",
-                "mnemonic": "W",
-                "id": "wakatime-settings",
+                "caption": "Package Settings",
+                "mnemonic": "P",
+                "id": "package-settings",
                 "children":
                 [
                     {
-                        "command": "open_file", "args":
-                        {
-                            "file": "${packages}/WakaTime/WakaTime.sublime-settings"
-                        },
-                        "caption": "Settings – Default"
-                    },
-                    {
-                        "command": "open_file", "args":
-                        {
-                            "file": "${packages}/User/WakaTime.sublime-settings"
-                        },
-                        "caption": "Settings – User"
+                        "caption": "WakaTime",
+                        "mnemonic": "W",
+                        "id": "wakatime-settings",
+                        "children":
+                        [
+                            {
+                                "command": "open_file", "args":
+                                {
+                                    "file": "${packages}/WakaTime/WakaTime.sublime-settings"
+                                },
+                                "caption": "Settings – Default"
+                            },
+                            {
+                                "command": "open_file", "args":
+                                {
+                                    "file": "${packages}/User/WakaTime.sublime-settings"
+                                },
+                                "caption": "Settings – User"
+                            },
+                            {
+                                "command": "wakatime_dashboard",
+                                "args": {},
+                                "caption": "WakaTime Dashboard"
+                            }
+                        ]
                     }
                 ]
             }

README.md

@@ -1,43 +1,59 @@
 sublime-wakatime
 ================
 
-Sublime Text 2 & 3 plugin to quantify your coding using https://wakatime.com/.
+Metrics, insights, and time tracking automatically generated from your programming activity.
+
 
 Installation
 ------------
 
-Heads Up! For Sublime Text 2 on Windows & Linux, WakaTime depends on [Python](http://www.python.org/getit/) being installed to work correctly.
-    
 1. Install [Package Control](https://packagecontrol.io/installation).
 
-2. Using [Package Control](https://packagecontrol.io/docs/usage):
+2. In Sublime, press `ctrl+shift+p`(Windows, Linux) or `cmd+shift+p`(OS X).
 
-  a) Inside Sublime, press `ctrl+shift+p`(Windows, Linux) or `cmd+shift+p`(OS X).
+3. Type `install`, then press `enter` with `Package Control: Install Package` selected.
 
-  b) Type `install`, then press `enter` with `Package Control: Install Package` selected.
+4. Type `wakatime`, then press `enter` with the `WakaTime` plugin selected.
 
-  c) Type `wakatime`, then press `enter` with the `WakaTime` plugin selected.
+5. Enter your [api key](https://wakatime.com/settings#apikey), then press `enter`.
 
-3. Enter your [api key](https://wakatime.com/settings#apikey), then press `enter`.
+6. Use Sublime and your coding activity will be displayed on your [WakaTime dashboard](https://wakatime.com).
 
-4. Use Sublime and your time will be tracked for you automatically.
-
-5. Visit https://wakatime.com/dashboard to see your logged time.
 
 Screen Shots
 ------------
 
-![Project Overview](https://wakatime.com/static/img/ScreenShots/ScreenShot-2014-10-29.png)
+![Project Overview](https://wakatime.com/static/img/ScreenShots/Screen-Shot-2016-03-21.png)
+
+
+Unresponsive Plugin Warning
+---------------------------
+
+In Sublime Text 2, if you get a warning message:
+
+    A plugin (WakaTime) may be making Sublime Text unresponsive by taking too long (0.017332s) in its on_modified callback.
+
+To fix this, go to `Preferences → Settings - User` then add the following setting:
+
+`"detect_slow_plugins": false`
+
 
 Troubleshooting
 ---------------
 
 First, turn on debug mode in your `WakaTime.sublime-settings` file.
 
-![sublime user settings](https://wakatime.com/static/img/ScreenShots/sublime-wakatime-settings-menu.png)
+![sublime user settings](https://wakatime.com/static/img/ScreenShots/sublime-wakatime-settings-menu.png?v=3)
 
 Add the line: `"debug": true`
 
-Then, open your Sublime Console with `View -> Show Console` to see the plugin executing the wakatime cli process when sending a heartbeat. Also, tail your `$HOME/.wakatime.log` file to debug wakatime cli problems.
+Then, open your Sublime Console with `View → Show Console` ( CTRL + \` ) to see the plugin executing the wakatime cli process when sending a heartbeat.
+Also, tail your `$HOME/.wakatime.log` file to debug wakatime cli problems.
 
-For more general troubleshooting information, see [wakatime/wakatime#troubleshooting](https://github.com/wakatime/wakatime#troubleshooting).
+The [How to Debug Plugins][how to debug] guide shows how to check when coding activity was last received from your editor using the [User Agents API][user agents api].
+For more general troubleshooting info, see the [wakatime-cli Troubleshooting Section][wakatime-cli-help].
+
+
+[wakatime-cli-help]: https://github.com/wakatime/wakatime#troubleshooting
+[how to debug]: https://wakatime.com/faq#debug-plugins
+[user agents api]: https://wakatime.com/developers#user_agents

WakaTime.py

@@ -7,22 +7,26 @@ Website:     https://wakatime.com/
 ==========================================================="""
 
 
-__version__ = '4.0.20'
+__version__ = '7.0.22'
 
 
 import sublime
 import sublime_plugin
 
+import contextlib
+import json
 import os
 import platform
 import re
 import sys
 import time
 import threading
+import traceback
 import urllib
 import webbrowser
 from datetime import datetime
-from subprocess import Popen
+from subprocess import Popen, STDOUT, PIPE
+from zipfile import ZipFile
 try:
     import _winreg as winreg  # py2
 except ImportError:
@@ -30,6 +34,61 @@ except ImportError:
         import winreg  # py3
     except ImportError:
         winreg = None
+try:
+    import Queue as queue  # py2
+except ImportError:
+    import queue  # py3
+
+
+is_py2 = (sys.version_info[0] == 2)
+is_py3 = (sys.version_info[0] == 3)
+
+if is_py2:
+    def u(text):
+        if text is None:
+            return None
+        if isinstance(text, unicode):
+            return text
+        try:
+            return text.decode('utf-8')
+        except:
+            try:
+                return text.decode(sys.getdefaultencoding())
+            except:
+                try:
+                    return unicode(text)
+                except:
+                    try:
+                        return text.decode('utf-8', 'replace')
+                    except:
+                        try:
+                            return unicode(str(text))
+                        except:
+                            return unicode('')
+
+elif is_py3:
+    def u(text):
+        if text is None:
+            return None
+        if isinstance(text, bytes):
+            try:
+                return text.decode('utf-8')
+            except:
+                try:
+                    return text.decode(sys.getdefaultencoding())
+                except:
+                    pass
+        try:
+            return str(text)
+        except:
+            return text.decode('utf-8', 'replace')
+
+else:
+    raise Exception('Unsupported Python version: {0}.{1}.{2}'.format(
+        sys.version_info[0],
+        sys.version_info[1],
+        sys.version_info[2],
+    ))
 
 
 # globals
@@ -44,40 +103,85 @@ LAST_HEARTBEAT = {
     'file': None,
     'is_write': False,
 }
-LOCK = threading.RLock()
 PYTHON_LOCATION = None
+HEARTBEATS = queue.Queue()
+
+
+# Log Levels
+DEBUG = 'DEBUG'
+INFO = 'INFO'
+WARNING = 'WARNING'
+ERROR = 'ERROR'
 
 
 # add wakatime package to path
 sys.path.insert(0, os.path.join(PLUGIN_DIR, 'packages'))
 try:
-    from wakatime.base import parseConfigFile
+    from wakatime.main import parseConfigFile
 except ImportError:
     pass
 
 
-def createConfigFile():
-    """Creates the .wakatime.cfg INI file in $HOME directory, if it does
-    not already exist.
+def set_timeout(callback, seconds):
+    """Runs the callback after the given seconds delay.
+
+    If this is Sublime Text 3, runs the callback on an alternate thread. If this
+    is Sublime Text 2, runs the callback in the main thread.
     """
-    configFile = os.path.join(os.path.expanduser('~'), '.wakatime.cfg')
+
+    milliseconds = int(seconds * 1000)
     try:
-        with open(configFile) as fh:
-            pass
-    except IOError:
+        sublime.set_timeout_async(callback, milliseconds)
+    except AttributeError:
+        sublime.set_timeout(callback, milliseconds)
+
+
+def log(lvl, message, *args, **kwargs):
+    try:
+        if lvl == DEBUG and not SETTINGS.get('debug'):
+            return
+        msg = message
+        if len(args) > 0:
+            msg = message.format(*args)
+        elif len(kwargs) > 0:
+            msg = message.format(**kwargs)
         try:
-            with open(configFile, 'w') as fh:
-                fh.write("[settings]\n")
-                fh.write("debug = false\n")
-                fh.write("hidefilenames = false\n")
-        except IOError:
-            pass
+            print('[WakaTime] [{lvl}] {msg}'.format(lvl=lvl, msg=msg))
+        except UnicodeDecodeError:
+            print(u('[WakaTime] [{lvl}] {msg}').format(lvl=lvl, msg=u(msg)))
+    except RuntimeError:
+        set_timeout(lambda: log(lvl, message, *args, **kwargs), 0)
+
+
+def resources_folder():
+    if platform.system() == 'Windows':
+        return os.path.join(os.getenv('APPDATA'), 'WakaTime')
+    else:
+        return os.path.join(os.path.expanduser('~'), '.wakatime')
+
+
+def update_status_bar(status):
+    """Updates the status bar."""
+
+    try:
+        if SETTINGS.get('status_bar_message'):
+            msg = datetime.now().strftime(SETTINGS.get('status_bar_message_fmt'))
+            if '{status}' in msg:
+                msg = msg.format(status=status)
+
+            active_window = sublime.active_window()
+            if active_window:
+                for view in active_window.views():
+                    view.set_status('wakatime', msg)
+    except RuntimeError:
+        set_timeout(lambda: update_status_bar(status), 0)
 
 
 def prompt_api_key():
     global SETTINGS
 
-    createConfigFile()
+    if SETTINGS.get('api_key'):
+        return True
 
     default_key = ''
     try:
@@ -88,29 +192,27 @@ def prompt_api_key():
     except:
         pass
 
-    if SETTINGS.get('api_key'):
-        return True
-    else:
+    window = sublime.active_window()
+    if window:
         def got_key(text):
             if text:
                 SETTINGS.set('api_key', str(text))
                 sublime.save_settings(SETTINGS_FILE)
-        window = sublime.active_window()
-        if window:
-            window.show_input_panel('[WakaTime] Enter your wakatime.com api key:', default_key, got_key, None, None)
-            return True
-        else:
-            print('[WakaTime] Error: Could not prompt for api key because no window found.')
-    return False
+        window.show_input_panel('[WakaTime] Enter your wakatime.com api key:', default_key, got_key, None, None)
+        return True
+    else:
+        log(ERROR, 'Could not prompt for api key because no window found.')
+        return False
 
 
 def python_binary():
-    global PYTHON_LOCATION
     if PYTHON_LOCATION is not None:
         return PYTHON_LOCATION
 
     # look for python in PATH and common install locations
     paths = [
+        os.path.join(resources_folder(), 'python'),
+        None,
         '/',
         '/usr/local/bin/',
         '/usr/bin/',
@@ -118,22 +220,28 @@ def python_binary():
     for path in paths:
         path = find_python_in_folder(path)
         if path is not None:
-            PYTHON_LOCATION = path
+            set_python_binary_location(path)
             return path
 
     # look for python in windows registry
     path = find_python_from_registry(r'SOFTWARE\Python\PythonCore')
     if path is not None:
-        PYTHON_LOCATION = path
+        set_python_binary_location(path)
         return path
     path = find_python_from_registry(r'SOFTWARE\Wow6432Node\Python\PythonCore')
     if path is not None:
-        PYTHON_LOCATION = path
+        set_python_binary_location(path)
         return path
 
     return None
 
 
+def set_python_binary_location(path):
+    global PYTHON_LOCATION
+    PYTHON_LOCATION = path
+    log(DEBUG, 'Found Python at: {0}'.format(path))
+
+
 def find_python_from_registry(location, reg=None):
     if platform.system() != 'Windows' or winreg is None:
         return None
@@ -168,36 +276,60 @@ def find_python_from_registry(location, reg=None):
                     if path is not None:
                         path = find_python_in_folder(path)
                         if path is not None:
+                            log(DEBUG, 'Found python from {reg}\\{key}\\{version}\\{sub_key}.'.format(
+                                reg=reg,
+                                key=location,
+                                version=version,
+                                sub_key=sub_key,
+                            ))
                             return path
                 except WindowsError:
-                    print('[WakaTime] Warning: Could not read registry value "{reg}\\{key}\\{version}\\{sub_key}".'.format(
-                        reg='HKEY_CURRENT_USER',
+                    log(DEBUG, 'Could not read registry value "{reg}\\{key}\\{version}\\{sub_key}".'.format(
+                        reg=reg,
                         key=location,
                         version=version,
                         sub_key=sub_key,
                     ))
     except WindowsError:
-        print('[WakaTime] Warning: Could not read registry value "{reg}\\{key}".'.format(
-            reg='HKEY_CURRENT_USER',
+        log(DEBUG, 'Could not read registry value "{reg}\\{key}".'.format(
+            reg=reg,
             key=location,
         ))
+    except:
+        log(ERROR, 'Could not read registry value "{reg}\\{key}":\n{exc}'.format(
+            reg=reg,
+            key=location,
+            exc=traceback.format_exc(),
+        ))
 
     return val
 
 
-def find_python_in_folder(folder):
-    path = os.path.realpath(os.path.join(folder, 'pythonw'))
+def find_python_in_folder(folder, headless=True):
+    pattern = re.compile(r'\d+\.\d+')
+
+    path = 'python'
+    if folder is not None:
+        path = os.path.realpath(os.path.join(folder, 'python'))
+    if headless:
+        path = u(path) + u('w')
+    log(DEBUG, u('Looking for Python at: {0}').format(u(path)))
     try:
-        Popen([path, '--version'])
-        return path
+        process = Popen([path, '--version'], stdout=PIPE, stderr=STDOUT)
+        output, err = process.communicate()
+        output = u(output).strip()
+        retcode = process.poll()
+        log(DEBUG, u('Python Version Output: {0}').format(output))
+        if not retcode and pattern.search(output):
+            return path
     except:
-        pass
-    path = os.path.realpath(os.path.join(folder, 'python'))
-    try:
-        Popen([path, '--version'])
-        return path
-    except:
-        pass
+        log(DEBUG, u(sys.exc_info()[1]))
+
+    if headless:
+        path = find_python_in_folder(folder, headless=False)
+        if path is not None:
+            return path
+
     return None
 
 
@@ -213,10 +345,10 @@ def obfuscate_apikey(command_list):
     return cmd
 
 
-def enough_time_passed(now, last_heartbeat, is_write):
-    if now - last_heartbeat['time'] > HEARTBEAT_FREQUENCY * 60:
+def enough_time_passed(now, is_write):
+    if now - LAST_HEARTBEAT['time'] > HEARTBEAT_FREQUENCY * 60:
         return True
-    if is_write and now - last_heartbeat['time'] > 2:
+    if is_write and now - LAST_HEARTBEAT['time'] > 2:
         return True
     return False
 
@@ -260,142 +392,241 @@ def is_view_active(view):
     return False
 
 
-def handle_heartbeat(view, is_write=False):
+def handle_activity(view, is_write=False):
     window = view.window()
     if window is not None:
-        target_file = view.file_name()
-        project = window.project_data() if hasattr(window, 'project_data') else None
-        folders = window.folders()
-        thread = SendHeartbeatThread(target_file, view, is_write=is_write, project=project, folders=folders)
-        thread.start()
+        entity = view.file_name()
+        if entity:
+            timestamp = time.time()
+            last_file = LAST_HEARTBEAT['file']
+            if entity != last_file or enough_time_passed(timestamp, is_write):
+                project = window.project_data() if hasattr(window, 'project_data') else None
+                folders = window.folders()
+                append_heartbeat(entity, timestamp, is_write, view, project, folders)
 
 
-class SendHeartbeatThread(threading.Thread):
+def append_heartbeat(entity, timestamp, is_write, view, project, folders):
+    global LAST_HEARTBEAT
+
+    # add this heartbeat to queue
+    heartbeat = {
+        'entity': entity,
+        'timestamp': timestamp,
+        'is_write': is_write,
+        'cursorpos': view.sel()[0].begin() if view.sel() else None,
+        'project': project,
+        'folders': folders,
+    }
+    HEARTBEATS.put_nowait(heartbeat)
+
+    # make this heartbeat the LAST_HEARTBEAT
+    LAST_HEARTBEAT = {
+        'file': entity,
+        'time': timestamp,
+        'is_write': is_write,
+    }
+
+    # process the queue of heartbeats in the future
+    seconds = 4
+    set_timeout(process_queue, seconds)
+
+
+def process_queue():
+    try:
+        heartbeat = HEARTBEATS.get_nowait()
+    except queue.Empty:
+        return
+
+    has_extra_heartbeats = False
+    extra_heartbeats = []
+    try:
+        while True:
+            extra_heartbeats.append(HEARTBEATS.get_nowait())
+            has_extra_heartbeats = True
+    except queue.Empty:
+        pass
+
+    thread = SendHeartbeatsThread(heartbeat)
+    if has_extra_heartbeats:
+        thread.add_extra_heartbeats(extra_heartbeats)
+    thread.start()
+
+
+class SendHeartbeatsThread(threading.Thread):
     """Non-blocking thread for sending heartbeats to api.
     """
 
-    def __init__(self, target_file, view, is_write=False, project=None, folders=None, force=False):
+    def __init__(self, heartbeat):
         threading.Thread.__init__(self)
-        self.lock = LOCK
-        self.target_file = target_file
-        self.is_write = is_write
-        self.project = project
-        self.folders = folders
-        self.force = force
+
         self.debug = SETTINGS.get('debug')
         self.api_key = SETTINGS.get('api_key', '')
         self.ignore = SETTINGS.get('ignore', [])
-        self.last_heartbeat = LAST_HEARTBEAT.copy()
-        self.cursorpos = view.sel()[0].begin() if view.sel() else None
-        self.view = view
+        self.hidefilenames = SETTINGS.get('hidefilenames')
+        self.proxy = SETTINGS.get('proxy')
+
+        self.heartbeat = heartbeat
+        self.has_extra_heartbeats = False
+
+    def add_extra_heartbeats(self, extra_heartbeats):
+        self.has_extra_heartbeats = True
+        self.extra_heartbeats = extra_heartbeats
 
     def run(self):
-        with self.lock:
-            if self.target_file:
-                self.timestamp = time.time()
-                if self.force or self.target_file != self.last_heartbeat['file'] or enough_time_passed(self.timestamp, self.last_heartbeat, self.is_write):
-                    self.send_heartbeat()
+        """Running in background thread."""
 
-    def send_heartbeat(self):
-        if not self.api_key:
-            print('[WakaTime] Error: missing api key.')
-            return
-        ua = 'sublime/%d sublime-wakatime/%s' % (ST_VERSION, __version__)
-        cmd = [
-            API_CLIENT,
-            '--file', self.target_file,
-            '--time', str('%f' % self.timestamp),
-            '--plugin', ua,
-            '--key', str(bytes.decode(self.api_key.encode('utf8'))),
-        ]
-        if self.is_write:
-            cmd.append('--write')
-        if self.project and self.project.get('name'):
-            cmd.extend(['--alternate-project', self.project.get('name')])
-        elif self.folders:
-            project_name = find_project_from_folders(self.folders, self.target_file)
-            if project_name:
-                cmd.extend(['--alternate-project', project_name])
-        if self.cursorpos is not None:
-            cmd.extend(['--cursorpos', '{0}'.format(self.cursorpos)])
-        for pattern in self.ignore:
-            cmd.extend(['--ignore', pattern])
-        if self.debug:
-            cmd.append('--verbose')
-        if python_binary():
-            cmd.insert(0, python_binary())
-            if self.debug:
-                print('[WakaTime] %s' % ' '.join(obfuscate_apikey(cmd)))
-            if platform.system() == 'Windows':
-                Popen(cmd, shell=False)
-            else:
-                with open(os.path.join(os.path.expanduser('~'), '.wakatime.log'), 'a') as stderr:
-                    Popen(cmd, stderr=stderr)
-            self.sent()
-        else:
-            print('[WakaTime] Error: Unable to find python binary.')
+        self.send_heartbeats()
 
-    def sent(self):
-        sublime.set_timeout(self.set_status_bar, 0)
-        sublime.set_timeout(self.set_last_heartbeat, 0)
+    def build_heartbeat(self, entity=None, timestamp=None, is_write=None,
+                        cursorpos=None, project=None, folders=None):
+        """Returns a dict for passing to wakatime-cli as arguments."""
 
-    def set_status_bar(self):
-        if SETTINGS.get('status_bar_message'):
-            self.view.set_status('wakatime', datetime.now().strftime(SETTINGS.get('status_bar_message_fmt')))
-
-    def set_last_heartbeat(self):
-        global LAST_HEARTBEAT
-        LAST_HEARTBEAT = {
-            'file': self.target_file,
-            'time': self.timestamp,
-            'is_write': self.is_write,
+        heartbeat = {
+            'entity': entity,
+            'timestamp': timestamp,
+            'is_write': is_write,
         }
 
+        if project and project.get('name'):
+            heartbeat['alternate_project'] = project.get('name')
+        elif folders:
+            project_name = find_project_from_folders(folders, entity)
+            if project_name:
+                heartbeat['alternate_project'] = project_name
 
-class InstallPython(threading.Thread):
-    """Non-blocking thread for installing Python on Windows machines.
+        if cursorpos is not None:
+            heartbeat['cursorpos'] = '{0}'.format(cursorpos)
+
+        return heartbeat
+
+    def send_heartbeats(self):
+        if python_binary():
+            heartbeat = self.build_heartbeat(**self.heartbeat)
+            ua = 'sublime/%d sublime-wakatime/%s' % (ST_VERSION, __version__)
+            cmd = [
+                python_binary(),
+                API_CLIENT,
+                '--entity', heartbeat['entity'],
+                '--time', str('%f' % heartbeat['timestamp']),
+                '--plugin', ua,
+            ]
+            if self.api_key:
+                cmd.extend(['--key', str(bytes.decode(self.api_key.encode('utf8')))])
+            if heartbeat['is_write']:
+                cmd.append('--write')
+            if heartbeat.get('alternate_project'):
+                cmd.extend(['--alternate-project', heartbeat['alternate_project']])
+            if heartbeat.get('cursorpos') is not None:
+                cmd.extend(['--cursorpos', heartbeat['cursorpos']])
+            for pattern in self.ignore:
+                cmd.extend(['--exclude', pattern])
+            if self.debug:
+                cmd.append('--verbose')
+            if self.hidefilenames:
+                cmd.append('--hidefilenames')
+            if self.proxy:
+                cmd.extend(['--proxy', self.proxy])
+            if self.has_extra_heartbeats:
+                cmd.append('--extra-heartbeats')
+                stdin = PIPE
+                extra_heartbeats = [self.build_heartbeat(**x) for x in self.extra_heartbeats]
+                extra_heartbeats = json.dumps(extra_heartbeats)
+            else:
+                extra_heartbeats = None
+                stdin = None
+
+            log(DEBUG, ' '.join(obfuscate_apikey(cmd)))
+            try:
+                process = Popen(cmd, stdin=stdin, stdout=PIPE, stderr=STDOUT)
+                inp = None
+                if self.has_extra_heartbeats:
+                    inp = "{0}\n".format(extra_heartbeats)
+                    inp = inp.encode('utf-8')
+                output, err = process.communicate(input=inp)
+                output = u(output)
+                retcode = process.poll()
+                if (not retcode or retcode == 102) and not output:
+                    self.sent()
+                else:
+                    update_status_bar('Error')
+                if retcode:
+                    log(DEBUG if retcode == 102 else ERROR, 'wakatime-core exited with status: {0}'.format(retcode))
+                if output:
+                    log(ERROR, u('wakatime-core output: {0}').format(output))
+            except:
+                log(ERROR, u(sys.exc_info()[1]))
+                update_status_bar('Error')
+
+        else:
+            log(ERROR, 'Unable to find python binary.')
+            update_status_bar('Error')
+
+    def sent(self):
+        update_status_bar('OK')
+
+
+def download_python():
+    thread = DownloadPython()
+    thread.start()
+
+
+class DownloadPython(threading.Thread):
+    """Non-blocking thread for extracting embeddable Python on Windows machines.
     """
 
     def run(self):
-        print('[WakaTime] Downloading and installing python...')
-        url = 'https://www.python.org/ftp/python/3.4.3/python-3.4.3.msi'
-        if platform.architecture()[0] == '64bit':
-            url = 'https://www.python.org/ftp/python/3.4.3/python-3.4.3.amd64.msi'
-        python_msi = os.path.join(os.path.expanduser('~'), 'python.msi')
+        log(INFO, 'Downloading embeddable Python...')
+
+        ver = '3.5.2'
+        arch = 'amd64' if platform.architecture()[0] == '64bit' else 'win32'
+        url = 'https://www.python.org/ftp/python/{ver}/python-{ver}-embed-{arch}.zip'.format(
+            ver=ver,
+            arch=arch,
+        )
+
+        if not os.path.exists(resources_folder()):
+            os.makedirs(resources_folder())
+
+        zip_file = os.path.join(resources_folder(), 'python.zip')
         try:
-            urllib.urlretrieve(url, python_msi)
+            urllib.urlretrieve(url, zip_file)
         except AttributeError:
-            urllib.request.urlretrieve(url, python_msi)
-        args = [
-            'msiexec',
-            '/i',
-            python_msi,
-            '/norestart',
-            '/qb!',
-        ]
-        Popen(args)
+            urllib.request.urlretrieve(url, zip_file)
+
+        log(INFO, 'Extracting Python...')
+        with contextlib.closing(ZipFile(zip_file)) as zf:
+            path = os.path.join(resources_folder(), 'python')
+            zf.extractall(path)
+
+        try:
+            os.remove(zip_file)
+        except:
+            pass
+
+        log(INFO, 'Finished extracting Python.')
 
 
 def plugin_loaded():
     global SETTINGS
-    print('[WakaTime] Initializing WakaTime plugin v%s' % __version__)
+    SETTINGS = sublime.load_settings(SETTINGS_FILE)
+
+    log(INFO, 'Initializing WakaTime plugin v%s' % __version__)
+    update_status_bar('Initializing')
 
     if not python_binary():
-        print('[WakaTime] Warning: Python binary not found.')
+        log(WARNING, 'Python binary not found.')
         if platform.system() == 'Windows':
-            thread = InstallPython()
-            thread.start()
+            set_timeout(download_python, 0)
         else:
             sublime.error_message("Unable to find Python binary!\nWakaTime needs Python to work correctly.\n\nGo to https://www.python.org/downloads")
             return
 
-    SETTINGS = sublime.load_settings(SETTINGS_FILE)
     after_loaded()
 
 
 def after_loaded():
     if not prompt_api_key():
-        sublime.set_timeout(after_loaded, 500)
+        set_timeout(after_loaded, 0.5)
 
 
 # need to call plugin_loaded because only ST3 will auto-call it
@@ -406,15 +637,15 @@ if ST_VERSION < 3000:
 class WakatimeListener(sublime_plugin.EventListener):
 
     def on_post_save(self, view):
-        handle_heartbeat(view, is_write=True)
+        handle_activity(view, is_write=True)
 
     def on_selection_modified(self, view):
         if is_view_active(view):
-            handle_heartbeat(view)
+            handle_activity(view)
 
     def on_modified(self, view):
         if is_view_active(view):
-            handle_heartbeat(view)
+            handle_activity(view)
 
 
 class WakatimeDashboardCommand(sublime_plugin.ApplicationCommand):

WakaTime.sublime-settings

@@ -6,18 +6,24 @@
     // Your api key from https://wakatime.com/#apikey
     // Set this in your User specific WakaTime.sublime-settings file.
     "api_key": "",
-    
-    // Ignore files; Files (including absolute paths) that match one of these
-    // POSIX regular expressions will not be logged.
-    "ignore": ["^/tmp/", "^/etc/", "^/var/", "COMMIT_EDITMSG$", "PULLREQ_EDITMSG$", "MERGE_MSG$", "TAG_EDITMSG$"],
-    
+
     // Debug mode. Set to true for verbose logging. Defaults to false.
     "debug": false,
-    
+
+    // Proxy with format https://user:pass@host:port or socks5://user:pass@host:port or domain\\user:pass.
+    "proxy": "",
+
+    // Ignore files; Files (including absolute paths) that match one of these
+    // POSIX regular expressions will not be logged.
+    "ignore": ["^/tmp/", "^/etc/", "^/var/(?!www/).*", "COMMIT_EDITMSG$", "PULLREQ_EDITMSG$", "MERGE_MSG$", "TAG_EDITMSG$"],
+
     // Status bar message. Set to false to hide status bar message.
     // Defaults to true.
     "status_bar_message": true,
-    
+
     // Status bar message format.
-    "status_bar_message_fmt": "WakaTime active %I:%M %p"
+    "status_bar_message_fmt": "WakaTime {status} %I:%M %p",
+
+    // Obfuscate file paths when sending to API. Your dashboard will no longer display coding activity per file.
+    "hidefilenames": false
 }

packages/wakatime/__about__.py

@@ -1,9 +1,9 @@
 __title__ = 'wakatime'
 __description__ = 'Common interface to the WakaTime api.'
 __url__ = 'https://github.com/wakatime/wakatime'
-__version_info__ = ('4', '1', '8')
+__version_info__ = ('8', '0', '3')
 __version__ = '.'.join(__version_info__)
 __author__ = 'Alan Hamlett'
 __author_email__ = 'alan@wakatime.com'
 __license__ = 'BSD'
-__copyright__ = 'Copyright 2014 Alan Hamlett'
+__copyright__ = 'Copyright 2017 Alan Hamlett'

packages/wakatime/arguments.py

@@ -0,0 +1,240 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.arguments
+    ~~~~~~~~~~~~~~~~~~
+
+    Command-line arguments.
+
+    :copyright: (c) 2016 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+
+from __future__ import print_function
+
+
+import os
+import re
+import time
+import traceback
+from .__about__ import __version__
+from .configs import parseConfigFile
+from .constants import AUTH_ERROR
+from .packages import argparse
+
+
+class FileAction(argparse.Action):
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        try:
+            if os.path.isfile(values):
+                values = os.path.realpath(values)
+        except:  # pragma: nocover
+            pass
+        setattr(namespace, self.dest, values)
+
+
+def parseArguments():
+    """Parse command line arguments and configs from ~/.wakatime.cfg.
+    Command line arguments take precedence over config file settings.
+    Returns instances of ArgumentParser and SafeConfigParser.
+    """
+
+    # define supported command line arguments
+    parser = argparse.ArgumentParser(
+            description='Common interface for the WakaTime api.')
+    parser.add_argument('--entity', dest='entity', metavar='FILE',
+            action=FileAction,
+            help='absolute path to file for the heartbeat; can also be a '+
+                 'url, domain, or app when --entity-type is not file')
+    parser.add_argument('--file', dest='file', action=FileAction,
+            help=argparse.SUPPRESS)
+    parser.add_argument('--key', dest='key',
+            help='your wakatime api key; uses api_key from '+
+                '~/.wakatime.cfg by default')
+    parser.add_argument('--write', dest='is_write',
+            action='store_true',
+            help='when set, tells api this heartbeat was triggered from '+
+                 'writing to a file')
+    parser.add_argument('--plugin', dest='plugin',
+            help='optional text editor plugin name and version '+
+                 'for User-Agent header')
+    parser.add_argument('--time', dest='timestamp', metavar='time',
+            type=float,
+            help='optional floating-point unix epoch timestamp; '+
+                 'uses current time by default')
+    parser.add_argument('--lineno', dest='lineno',
+            help='optional line number; current line being edited')
+    parser.add_argument('--cursorpos', dest='cursorpos',
+            help='optional cursor position in the current file')
+    parser.add_argument('--entity-type', dest='entity_type',
+            help='entity type for this heartbeat. can be one of "file", '+
+                 '"domain", or "app"; defaults to file.')
+    parser.add_argument('--proxy', dest='proxy',
+                        help='optional proxy configuration. Supports HTTPS '+
+                        'and SOCKS proxies. For example: '+
+                        'https://user:pass@host:port or '+
+                        'socks5://user:pass@host:port or ' +
+                        'domain\\user:pass')
+    parser.add_argument('--no-ssl-verify', dest='nosslverify',
+                        action='store_true',
+                        help='disables SSL certificate verification for HTTPS '+
+                        'requests. By default, SSL certificates are verified.')
+    parser.add_argument('--project', dest='project',
+            help='optional project name')
+    parser.add_argument('--alternate-project', dest='alternate_project',
+            help='optional alternate project name; auto-discovered project '+
+                 'takes priority')
+    parser.add_argument('--alternate-language',  dest='alternate_language',
+            help=argparse.SUPPRESS)
+    parser.add_argument('--language', dest='language',
+            help='optional language name; if valid, takes priority over '+
+                 'auto-detected language')
+    parser.add_argument('--hostname', dest='hostname', help='hostname of '+
+                        'current machine.')
+    parser.add_argument('--disableoffline', dest='offline',
+            action='store_false',
+            help='disables offline time logging instead of queuing logged time')
+    parser.add_argument('--hidefilenames', dest='hidefilenames',
+            action='store_true',
+            help='obfuscate file names; will not send file names to api')
+    parser.add_argument('--exclude', dest='exclude', action='append',
+            help='filename patterns to exclude from logging; POSIX regex '+
+                 'syntax; can be used more than once')
+    parser.add_argument('--include', dest='include', action='append',
+            help='filename patterns to log; when used in combination with '+
+                 '--exclude, files matching include will still be logged; '+
+                 'POSIX regex syntax; can be used more than once')
+    parser.add_argument('--ignore', dest='ignore', action='append',
+            help=argparse.SUPPRESS)
+    parser.add_argument('--extra-heartbeats', dest='extra_heartbeats',
+            action='store_true',
+            help='reads extra heartbeats from STDIN as a JSON array until EOF')
+    parser.add_argument('--logfile', dest='logfile',
+            help='defaults to ~/.wakatime.log')
+    parser.add_argument('--apiurl', dest='api_url',
+            help='heartbeats api url; for debugging with a local server')
+    parser.add_argument('--timeout', dest='timeout', type=int,
+            help='number of seconds to wait when sending heartbeats to api; '+
+                 'defaults to 60 seconds')
+    parser.add_argument('--config', dest='config',
+            help='defaults to ~/.wakatime.cfg')
+    parser.add_argument('--verbose', dest='verbose', action='store_true',
+            help='turns on debug messages in log file')
+    parser.add_argument('--version', action='version', version=__version__)
+
+    # parse command line arguments
+    args = parser.parse_args()
+
+    # use current unix epoch timestamp by default
+    if not args.timestamp:
+        args.timestamp = time.time()
+
+    # parse ~/.wakatime.cfg file
+    configs = parseConfigFile(args.config)
+
+    # update args from configs
+    if not args.hostname:
+        if configs.has_option('settings', 'hostname'):
+            args.hostname = configs.get('settings', 'hostname')
+    if not args.key:
+        default_key = None
+        if configs.has_option('settings', 'api_key'):
+            default_key = configs.get('settings', 'api_key')
+        elif configs.has_option('settings', 'apikey'):
+            default_key = configs.get('settings', 'apikey')
+        if default_key:
+            args.key = default_key
+        else:
+            try:
+                parser.error('Missing api key. Find your api key from wakatime.com/settings.')
+            except SystemExit:
+                raise SystemExit(AUTH_ERROR)
+
+    is_valid = not not re.match(r'^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$', args.key, re.I)
+    if not is_valid:
+        try:
+            parser.error('Invalid api key. Find your api key from wakatime.com/settings.')
+        except SystemExit:
+            raise SystemExit(AUTH_ERROR)
+
+    if not args.entity:
+        if args.file:
+            args.entity = args.file
+        else:
+            parser.error('argument --entity is required')
+
+    if not args.language and args.alternate_language:
+        args.language = args.alternate_language
+
+    if not args.exclude:
+        args.exclude = []
+    if configs.has_option('settings', 'ignore'):
+        try:
+            for pattern in configs.get('settings', 'ignore').split("\n"):
+                if pattern.strip() != '':
+                    args.exclude.append(pattern)
+        except TypeError:  # pragma: nocover
+            pass
+    if configs.has_option('settings', 'exclude'):
+        try:
+            for pattern in configs.get('settings', 'exclude').split("\n"):
+                if pattern.strip() != '':
+                    args.exclude.append(pattern)
+        except TypeError:  # pragma: nocover
+            pass
+    if not args.include:
+        args.include = []
+    if configs.has_option('settings', 'include'):
+        try:
+            for pattern in configs.get('settings', 'include').split("\n"):
+                if pattern.strip() != '':
+                    args.include.append(pattern)
+        except TypeError:  # pragma: nocover
+            pass
+    if args.hidefilenames:
+        args.hidefilenames = ['.*']
+    else:
+        args.hidefilenames = []
+        if configs.has_option('settings', 'hidefilenames'):
+            option = configs.get('settings', 'hidefilenames')
+            if option.strip().lower() == 'true':
+                args.hidefilenames = ['.*']
+            elif option.strip().lower() != 'false':
+                for pattern in option.split("\n"):
+                    if pattern.strip() != '':
+                        args.hidefilenames.append(pattern)
+    if args.offline and configs.has_option('settings', 'offline'):
+        args.offline = configs.getboolean('settings', 'offline')
+    if not args.proxy and configs.has_option('settings', 'proxy'):
+        args.proxy = configs.get('settings', 'proxy')
+    if args.proxy:
+        pattern = r'^((https?|socks5)://)?([^:@]+(:([^:@])+)?@)?[^:]+(:\d+)?$'
+        if '\\' in args.proxy:
+            pattern = r'^.*\\.+$'
+        is_valid = not not re.match(pattern, args.proxy, re.I)
+        if not is_valid:
+            parser.error('Invalid proxy. Must be in format ' +
+                            'https://user:pass@host:port or ' +
+                            'socks5://user:pass@host:port or ' +
+                            'domain\\user:pass.')
+    if configs.has_option('settings', 'no_ssl_verify'):
+        args.nosslverify = configs.getboolean('settings', 'no_ssl_verify')
+    if not args.verbose and configs.has_option('settings', 'verbose'):
+        args.verbose = configs.getboolean('settings', 'verbose')
+    if not args.verbose and configs.has_option('settings', 'debug'):
+        args.verbose = configs.getboolean('settings', 'debug')
+    if not args.logfile and configs.has_option('settings', 'logfile'):
+        args.logfile = configs.get('settings', 'logfile')
+    if not args.logfile and os.environ.get('WAKATIME_HOME'):
+        home = os.environ.get('WAKATIME_HOME')
+        args.logfile = os.path.join(os.path.expanduser(home), '.wakatime.log')
+    if not args.api_url and configs.has_option('settings', 'api_url'):
+        args.api_url = configs.get('settings', 'api_url')
+    if not args.timeout and configs.has_option('settings', 'timeout'):
+        try:
+            args.timeout = int(configs.get('settings', 'timeout'))
+        except ValueError:
+            print(traceback.format_exc())
+
+    return args, configs

packages/wakatime/compat.py

@@ -31,7 +31,7 @@ if is_py2:  # pragma: nocover
                 try:
                     return unicode(text)
                 except:
-                    return text
+                    return text.decode('utf-8', 'replace')
     open = codecs.open
     basestring = basestring
 
@@ -52,7 +52,7 @@ elif is_py3:  # pragma: nocover
         try:
             return str(text)
         except:
-            return text
+            return text.decode('utf-8', 'replace')
     open = open
     basestring = (str, bytes)
 

packages/wakatime/configs.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.configs
+    ~~~~~~~~~~~~~~~~
+
+    Config file parser.
+
+    :copyright: (c) 2016 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+
+from __future__ import print_function
+
+import os
+import traceback
+
+from .compat import open
+from .constants import CONFIG_FILE_PARSE_ERROR
+
+
+try:
+    import configparser
+except ImportError:
+    from .packages import configparser
+
+
+def getConfigFile():
+    """Returns the config file location.
+
+    If $WAKATIME_HOME env varialbe is defined, returns
+    $WAKATIME_HOME/.wakatime.cfg, otherwise ~/.wakatime.cfg.
+    """
+
+    fileName = '.wakatime.cfg'
+
+    home = os.environ.get('WAKATIME_HOME')
+    if home:
+        return os.path.join(os.path.expanduser(home), fileName)
+
+    return os.path.join(os.path.expanduser('~'), fileName)
+
+
+def parseConfigFile(configFile=None):
+    """Returns a configparser.SafeConfigParser instance with configs
+    read from the config file. Default location of the config file is
+    at ~/.wakatime.cfg.
+    """
+
+    # get config file location from ENV
+    if not configFile:
+        configFile = getConfigFile()
+
+    configs = configparser.ConfigParser(delimiters=('='), strict=False)
+    try:
+        with open(configFile, 'r', encoding='utf-8') as fh:
+            try:
+                configs.read_file(fh)
+            except configparser.Error:
+                print(traceback.format_exc())
+                raise SystemExit(CONFIG_FILE_PARSE_ERROR)
+    except IOError:
+        pass
+    return configs

packages/wakatime/constants.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.constants
+    ~~~~~~~~~~~~~~~~~~
+
+    Constant variable definitions.
+
+    :copyright: (c) 2016 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+""" Success
+Exit code used when a heartbeat was sent successfully.
+"""
+SUCCESS = 0
+
+""" Api Error
+Exit code used when the WakaTime API returned an error.
+"""
+API_ERROR = 102
+
+""" Config File Parse Error
+Exit code used when the ~/.wakatime.cfg config file could not be parsed.
+"""
+CONFIG_FILE_PARSE_ERROR = 103
+
+""" Auth Error
+Exit code used when our api key is invalid.
+"""
+AUTH_ERROR = 104
+
+""" Unknown Error
+Exit code used when there was an unhandled exception.
+"""
+UNKNOWN_ERROR = 105
+
+""" Malformed Heartbeat Error
+Exit code used when the JSON input from `--extra-heartbeats` is malformed.
+"""
+MALFORMED_HEARTBEAT_ERROR = 106
+
+""" Connection Error
+Exit code used when there was proxy or other problem connecting to the WakaTime
+API servers.
+"""
+CONNECTION_ERROR = 107
+
+""" Max file size supporting line number count stats.
+Files larger than this in bytes will not have a line count stat for performance.
+Default is 2MB.
+"""
+MAX_FILE_SIZE_SUPPORTED = 2000000

packages/wakatime/dependencies/__init__.py

@@ -12,7 +12,6 @@
 import logging
 import re
 import sys
-import traceback
 
 from ..compat import u, open, import_module
 from ..exceptions import NotYetImplemented
@@ -68,7 +67,7 @@ class TokenParser(object):
                 pass
             try:
                 with open(self.source_file, 'r', encoding=sys.getfilesystemencoding()) as fh:
-                    return self.lexer.get_tokens_unprocessed(fh.read(512000))
+                    return self.lexer.get_tokens_unprocessed(fh.read(512000))  # pragma: nocover
             except:
                 pass
         return []
@@ -118,9 +117,9 @@ class DependencyParser(object):
             try:
                 self.parser = getattr(module, class_name)
             except AttributeError:
-                log.debug('Module {0} is missing class {1}'.format(module.__name__, class_name))
+                log.debug('Parsing dependencies not supported for {0}.{1}'.format(module_name, class_name))
         except ImportError:
-            log.debug(traceback.format_exc())
+            log.debug('Parsing dependencies not supported for {0}.{1}'.format(module_name, class_name))
 
     def parse(self):
         if self.parser:

packages/wakatime/dependencies/c_cpp.py

@@ -12,34 +12,6 @@
 from . import TokenParser
 
 
-class CppParser(TokenParser):
-    exclude = [
-        r'^stdio\.h$',
-        r'^stdlib\.h$',
-        r'^string\.h$',
-        r'^time\.h$',
-    ]
-
-    def parse(self):
-        for index, token, content in self.tokens:
-            self._process_token(token, content)
-        return self.dependencies
-
-    def _process_token(self, token, content):
-        if self.partial(token) == 'Preproc':
-            self._process_preproc(token, content)
-        else:
-            self._process_other(token, content)
-
-    def _process_preproc(self, token, content):
-        if content.strip().startswith('include ') or content.strip().startswith("include\t"):
-            content = content.replace('include', '', 1).strip().strip('"').strip('<').strip('>').strip()
-            self.append(content)
-
-    def _process_other(self, token, content):
-        pass
-
-
 class CParser(TokenParser):
     exclude = [
         r'^stdio\.h$',
@@ -47,6 +19,7 @@ class CParser(TokenParser):
         r'^string\.h$',
         r'^time\.h$',
     ]
+    state = None
 
     def parse(self):
         for index, token, content in self.tokens:
@@ -54,15 +27,25 @@ class CParser(TokenParser):
         return self.dependencies
 
     def _process_token(self, token, content):
-        if self.partial(token) == 'Preproc':
+        if self.partial(token) == 'Preproc' or self.partial(token) == 'PreprocFile':
             self._process_preproc(token, content)
         else:
             self._process_other(token, content)
 
     def _process_preproc(self, token, content):
-        if content.strip().startswith('include ') or content.strip().startswith("include\t"):
-            content = content.replace('include', '', 1).strip().strip('"').strip('<').strip('>').strip()
-            self.append(content)
+        if self.state == 'include':
+            if content != '\n' and content != '#':
+                content = content.strip().strip('"').strip('<').strip('>').strip()
+                self.append(content, truncate=True, separator='/')
+            self.state = None
+        elif content.strip().startswith('include'):
+            self.state = 'include'
+        else:
+            self.state = None
 
     def _process_other(self, token, content):
         pass
+
+
+class CppParser(CParser):
+    pass

packages/wakatime/dependencies/go.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.languages.go
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Go code.
+
+    :copyright: (c) 2016 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+
+
+class GoParser(TokenParser):
+    state = None
+    parens = 0
+    aliases = 0
+    exclude = [
+        r'^"fmt"$',
+    ]
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        elif self.partial(token) == 'Punctuation':
+            self._process_punctuation(token, content)
+        elif self.partial(token) == 'String':
+            self._process_string(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        elif self.partial(token) == 'Other':
+            self._process_other(token, content)
+        else:
+            self._process_misc(token, content)
+
+    def _process_namespace(self, token, content):
+        self.state = content
+        self.parens = 0
+        self.aliases = 0
+
+    def _process_string(self, token, content):
+        if self.state == 'import':
+            self.append(content, truncate=False)
+
+    def _process_punctuation(self, token, content):
+        if content == '(':
+            self.parens += 1
+        elif content == ')':
+            self.parens -= 1
+        elif content == '.':
+            self.aliases += 1
+        else:
+            self.state = None
+
+    def _process_text(self, token, content):
+        if self.state == 'import':
+            if content == "\n" and self.parens <= 0:
+                self.state = None
+                self.parens = 0
+                self.aliases = 0
+        else:
+            self.state = None
+
+    def _process_other(self, token, content):
+        if self.state == 'import':
+            self.aliases += 1
+        else:
+            self.state = None
+
+    def _process_misc(self, token, content):
+        self.state = None

packages/wakatime/dependencies/python.py

@@ -18,7 +18,9 @@ class PythonParser(TokenParser):
     nonpackage = False
     exclude = [
         r'^os$',
+        r'^sys$',
         r'^sys\.',
+        r'^__future__$',
     ]
 
     def parse(self):
@@ -48,9 +50,7 @@ class PythonParser(TokenParser):
                 self._process_import(token, content)
 
     def _process_operator(self, token, content):
-        if self.state is not None:
-            if content == '.':
-                self.nonpackage = True
+        pass
 
     def _process_punctuation(self, token, content):
         if content == '(':
@@ -73,8 +73,6 @@ class PythonParser(TokenParser):
             if self.state == 'from':
                 self.append(content, truncate=True, truncate_to=1)
                 self.state = 'from-2'
-            elif self.state == 'from-2' and content != 'import':
-                self.append(content, truncate=True, truncate_to=1)
             elif self.state == 'import':
                 self.append(content, truncate=True, truncate_to=1)
                 self.state = 'import-2'

packages/wakatime/dependencies/templates.py

@@ -69,30 +69,9 @@ KEYWORDS = [
 ]
 
 
-class LassoJavascriptParser(TokenParser):
-
-    def parse(self):
-        for index, token, content in self.tokens:
-            self._process_token(token, content)
-        return self.dependencies
-
-    def _process_token(self, token, content):
-        if u(token) == 'Token.Name.Other':
-            self._process_name(token, content)
-        elif u(token) == 'Token.Literal.String.Single' or u(token) == 'Token.Literal.String.Double':
-            self._process_literal_string(token, content)
-
-    def _process_name(self, token, content):
-        if content.lower() in KEYWORDS:
-            self.append(content.lower())
-
-    def _process_literal_string(self, token, content):
-        if 'famous/core/' in content.strip('"').strip("'"):
-            self.append('famous')
-
-
 class HtmlDjangoParser(TokenParser):
     tags = []
+    opening_tag = False
     getting_attrs = False
     current_attr = None
     current_attr_value = None
@@ -103,7 +82,9 @@ class HtmlDjangoParser(TokenParser):
         return self.dependencies
 
     def _process_token(self, token, content):
-        if u(token) == 'Token.Name.Tag':
+        if u(token) == 'Token.Punctuation':
+            self._process_punctuation(token, content)
+        elif u(token) == 'Token.Name.Tag':
             self._process_tag(token, content)
         elif u(token) == 'Token.Literal.String':
             self._process_string(token, content)
@@ -114,26 +95,30 @@ class HtmlDjangoParser(TokenParser):
     def current_tag(self):
         return None if len(self.tags) == 0 else self.tags[0]
 
-    def _process_tag(self, token, content):
+    def _process_punctuation(self, token, content):
         if content.startswith('</') or content.startswith('/'):
             try:
                 self.tags.pop(0)
             except IndexError:
                 # ignore errors from malformed markup
                 pass
+            self.opening_tag = False
             self.getting_attrs = False
         elif content.startswith('<'):
+            self.opening_tag = True
+        elif content.startswith('>'):
+            self.opening_tag = False
+            self.getting_attrs = False
+
+    def _process_tag(self, token, content):
+        if self.opening_tag:
             self.tags.insert(0, content.replace('<', '', 1).strip().lower())
             self.getting_attrs = True
-        elif content.startswith('>'):
-            self.getting_attrs = False
         self.current_attr = None
 
     def _process_attribute(self, token, content):
         if self.getting_attrs:
             self.current_attr = content.lower().strip('=')
-        else:
-            self.current_attr = None
         self.current_attr_value = None
 
     def _process_string(self, token, content):
@@ -156,8 +141,6 @@ class HtmlDjangoParser(TokenParser):
             elif content.startswith('"') or content.startswith("'"):
                 if self.current_attr_value is None:
                     self.current_attr_value = content
-                else:
-                    self.current_attr_value += content
 
 
 class VelocityHtmlParser(HtmlDjangoParser):

packages/wakatime/language_priorities.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.language_priorities
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Overwrite pygments Lexer.priority attribute for specific languages.
+
+    :copyright: (c) 2017 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+
+LANGUAGES = {
+    'f#': 0.01,
+    'perl': 0.01,
+    'perl6': 0.01,
+    'typescript': 0.01,
+}

packages/wakatime/languages/default.json

@@ -0,0 +1,81 @@
+{
+  "actionscript": "ActionScript", 
+  "apacheconf": "ApacheConf", 
+  "applescript": "AppleScript", 
+  "asp": "ASP", 
+  "assembly": "Assembly", 
+  "awk": "Awk", 
+  "bash": "Bash", 
+  "basic": "Basic", 
+  "brightscript": "BrightScript", 
+  "c": "C", 
+  "c#": "C#", 
+  "c++": "C++", 
+  "clojure": "Clojure", 
+  "cocoa": "Cocoa", 
+  "coffeescript": "CoffeeScript", 
+  "coldfusion": "ColdFusion", 
+  "common lisp": "Common Lisp", 
+  "cshtml": "CSHTML", 
+  "css": "CSS", 
+  "dart": "Dart", 
+  "delphi": "Delphi", 
+  "elixir": "Elixir", 
+  "elm": "Elm", 
+  "emacs lisp": "Emacs Lisp", 
+  "erlang": "Erlang", 
+  "f#": "F#", 
+  "fortran": "Fortran", 
+  "go": "Go", 
+  "gous": "Gosu", 
+  "groovy": "Groovy", 
+  "haml": "Haml", 
+  "haskell": "Haskell", 
+  "haxe": "Haxe", 
+  "html": "HTML", 
+  "ini": "INI", 
+  "jade": "Jade", 
+  "java": "Java", 
+  "javascript": "JavaScript", 
+  "json": "JSON", 
+  "jsx": "JSX", 
+  "kotlin": "Kotlin", 
+  "less": "LESS", 
+  "lua": "Lua", 
+  "markdown": "Markdown", 
+  "matlab": "Matlab", 
+  "mustache": "Mustache", 
+  "objective-c": "Objective-C", 
+  "objective-c++": "Objective-C++", 
+  "objective-j": "Objective-J", 
+  "ocaml": "OCaml", 
+  "perl": "Perl", 
+  "php": "PHP", 
+  "powershell": "PowerShell", 
+  "prolog": "Prolog", 
+  "puppet": "Puppet", 
+  "python": "Python", 
+  "r": "R", 
+  "restructuredtext": "reStructuredText", 
+  "ruby": "Ruby", 
+  "rust": "Rust", 
+  "sass": "Sass", 
+  "scala": "Scala", 
+  "scheme": "Scheme", 
+  "scss": "SCSS", 
+  "shell": "Shell", 
+  "slim": "Slim", 
+  "smalltalk": "Smalltalk", 
+  "sql": "SQL", 
+  "swift": "Swift", 
+  "text": "Text", 
+  "turing": "Turing", 
+  "twig": "Twig", 
+  "typescript": "TypeScript", 
+  "typoscript": "TypoScript", 
+  "vb.net": "VB.net", 
+  "viml": "VimL", 
+  "xaml": "XAML", 
+  "xml": "XML", 
+  "yaml": "YAML"
+}

packages/wakatime/languages/vim.json

@@ -0,0 +1,531 @@
+{
+  "a2ps": null,
+  "a65": "Assembly",
+  "aap": null,
+  "abap": null,
+  "abaqus": null,
+  "abc": null,
+  "abel": null,
+  "acedb": null,
+  "ada": null,
+  "aflex": null,
+  "ahdl": null,
+  "alsaconf": null,
+  "amiga": null,
+  "aml": null,
+  "ampl": null,
+  "ant": null,
+  "antlr": null,
+  "apache": null,
+  "apachestyle": null,
+  "arch": null,
+  "art": null,
+  "asm": "Assembly",
+  "asm68k": "Assembly",
+  "asmh8300": "Assembly",
+  "asn": null,
+  "aspperl": null,
+  "aspvbs": null,
+  "asterisk": null,
+  "asteriskvm": null,
+  "atlas": null,
+  "autohotkey": null,
+  "autoit": null,
+  "automake": null,
+  "ave": null,
+  "awk": null,
+  "ayacc": null,
+  "b": null,
+  "baan": null,
+  "basic": "Basic",
+  "bc": null,
+  "bdf": null,
+  "bib": null,
+  "bindzone": null,
+  "blank": null,
+  "bst": null,
+  "btm": null,
+  "bzr": null,
+  "c": "C",
+  "cabal": null,
+  "calendar": null,
+  "catalog": null,
+  "cdl": null,
+  "cdrdaoconf": null,
+  "cdrtoc": null,
+  "cf": null,
+  "cfg": null,
+  "ch": null,
+  "chaiscript": null,
+  "change": null,
+  "changelog": null,
+  "chaskell": null,
+  "cheetah": null,
+  "chill": null,
+  "chordpro": null,
+  "cl": null,
+  "clean": null,
+  "clipper": null,
+  "cmake": null,
+  "cmusrc": null,
+  "cobol": null,
+  "coco": null,
+  "conaryrecipe": null,
+  "conf": null,
+  "config": null,
+  "context": null,
+  "cpp": "C++",
+  "crm": null,
+  "crontab": "Crontab",
+  "cs": "C#",
+  "csc": null,
+  "csh": null,
+  "csp": null,
+  "css": null,
+  "cterm": null,
+  "ctrlh": null,
+  "cucumber": null,
+  "cuda": null,
+  "cupl": null,
+  "cuplsim": null,
+  "cvs": null,
+  "cvsrc": null,
+  "cweb": null,
+  "cynlib": null,
+  "cynpp": null,
+  "d": null,
+  "datascript": null,
+  "dcd": null,
+  "dcl": null,
+  "debchangelog": null,
+  "debcontrol": null,
+  "debsources": null,
+  "def": null,
+  "denyhosts": null,
+  "desc": null,
+  "desktop": null,
+  "dictconf": null,
+  "dictdconf": null,
+  "diff": null,
+  "dircolors": null,
+  "diva": null,
+  "django": null,
+  "dns": null,
+  "docbk": null,
+  "docbksgml": null,
+  "docbkxml": null,
+  "dosbatch": null,
+  "dosini": null,
+  "dot": null,
+  "doxygen": null,
+  "dracula": null,
+  "dsl": null,
+  "dtd": null,
+  "dtml": null,
+  "dtrace": null,
+  "dylan": null,
+  "dylanintr": null,
+  "dylanlid": null,
+  "ecd": null,
+  "edif": null,
+  "eiffel": null,
+  "elf": null,
+  "elinks": null,
+  "elmfilt": null,
+  "erlang": null,
+  "eruby": null,
+  "esmtprc": null,
+  "esqlc": null,
+  "esterel": null,
+  "eterm": null,
+  "eviews": null,
+  "exim": null,
+  "expect": null,
+  "exports": null,
+  "fan": null,
+  "fasm": null,
+  "fdcc": null,
+  "fetchmail": null,
+  "fgl": null,
+  "flexwiki": null,
+  "focexec": null,
+  "form": null,
+  "forth": null,
+  "fortran": null,
+  "foxpro": null,
+  "framescript": null,
+  "freebasic": null,
+  "fstab": null,
+  "fvwm": null,
+  "fvwm2m4": null,
+  "gdb": null,
+  "gdmo": null,
+  "gedcom": null,
+  "git": null,
+  "gitcommit": null,
+  "gitconfig": null,
+  "gitrebase": null,
+  "gitsendemail": null,
+  "gkrellmrc": null,
+  "gnuplot": null,
+  "gp": null,
+  "gpg": null,
+  "grads": null,
+  "gretl": null,
+  "groff": null,
+  "groovy": null,
+  "group": null,
+  "grub": null,
+  "gsp": null,
+  "gtkrc": null,
+  "haml": "Haml",
+  "hamster": null,
+  "haskell": "Haskell",
+  "haste": null,
+  "hastepreproc": null,
+  "hb": null,
+  "help": null,
+  "hercules": null,
+  "hex": null,
+  "hog": null,
+  "hostconf": null,
+  "html": "HTML",
+  "htmlcheetah": "HTML",
+  "htmldjango": "HTML",
+  "htmlm4": "HTML",
+  "htmlos": null,
+  "ia64": null,
+  "ibasic": null,
+  "icemenu": null,
+  "icon": null,
+  "idl": null,
+  "idlang": null,
+  "indent": null,
+  "inform": null,
+  "initex": null,
+  "initng": null,
+  "inittab": null,
+  "ipfilter": null,
+  "ishd": null,
+  "iss": null,
+  "ist": null,
+  "jal": null,
+  "jam": null,
+  "jargon": null,
+  "java": "Java",
+  "javacc": null,
+  "javascript": "JavaScript",
+  "jess": null,
+  "jgraph": null,
+  "jproperties": null,
+  "jsp": null,
+  "kconfig": null,
+  "kix": null,
+  "kscript": null,
+  "kwt": null,
+  "lace": null,
+  "latte": null,
+  "ld": null,
+  "ldapconf": null,
+  "ldif": null,
+  "lex": null,
+  "lftp": null,
+  "lhaskell": "Haskell",
+  "libao": null,
+  "lifelines": null,
+  "lilo": null,
+  "limits": null,
+  "liquid": null,
+  "lisp": null,
+  "lite": null,
+  "litestep": null,
+  "loginaccess": null,
+  "logindefs": null,
+  "logtalk": null,
+  "lotos": null,
+  "lout": null,
+  "lpc": null,
+  "lprolog": null,
+  "lscript": null,
+  "lsl": null,
+  "lss": null,
+  "lua": null,
+  "lynx": null,
+  "m4": null,
+  "mail": null,
+  "mailaliases": null,
+  "mailcap": null,
+  "make": null,
+  "man": null,
+  "manconf": null,
+  "manual": null,
+  "maple": null,
+  "markdown": "Markdown",
+  "masm": null,
+  "mason": null,
+  "master": null,
+  "matlab": null,
+  "maxima": null,
+  "mel": null,
+  "messages": null,
+  "mf": null,
+  "mgl": null,
+  "mgp": null,
+  "mib": null,
+  "mma": null,
+  "mmix": null,
+  "mmp": null,
+  "modconf": null,
+  "model": null,
+  "modsim3": null,
+  "modula2": null,
+  "modula3": null,
+  "monk": null,
+  "moo": null,
+  "mp": null,
+  "mplayerconf": null,
+  "mrxvtrc": null,
+  "msidl": null,
+  "msmessages": null,
+  "msql": null,
+  "mupad": null,
+  "mush": null,
+  "muttrc": null,
+  "mysql": null,
+  "named": null,
+  "nanorc": null,
+  "nasm": null,
+  "nastran": null,
+  "natural": null,
+  "ncf": null,
+  "netrc": null,
+  "netrw": null,
+  "nosyntax": null,
+  "nqc": null,
+  "nroff": null,
+  "nsis": null,
+  "obj": null,
+  "objc": "Objective-C",
+  "objcpp": "Objective-C++",
+  "ocaml": "OCaml",
+  "occam": null,
+  "omnimark": null,
+  "openroad": null,
+  "opl": null,
+  "ora": null,
+  "pamconf": null,
+  "papp": null,
+  "pascal": null,
+  "passwd": null,
+  "pcap": null,
+  "pccts": null,
+  "pdf": null,
+  "perl": "Perl",
+  "perl6": "Perl",
+  "pf": null,
+  "pfmain": null,
+  "php": "PHP",
+  "phtml": "PHP",
+  "pic": null,
+  "pike": null,
+  "pilrc": null,
+  "pine": null,
+  "pinfo": null,
+  "plaintex": null,
+  "plm": null,
+  "plp": null,
+  "plsql": null,
+  "po": null,
+  "pod": null,
+  "postscr": null,
+  "pov": null,
+  "povini": null,
+  "ppd": null,
+  "ppwiz": null,
+  "prescribe": null,
+  "privoxy": null,
+  "procmail": null,
+  "progress": null,
+  "prolog": "Prolog",
+  "promela": null,
+  "protocols": null,
+  "psf": null,
+  "ptcap": null,
+  "purifylog": null,
+  "pyrex": null,
+  "python": "Python",
+  "qf": null,
+  "quake": null,
+  "r": "R",
+  "racc": null,
+  "radiance": null,
+  "ratpoison": null,
+  "rc": null,
+  "rcs": null,
+  "rcslog": null,
+  "readline": null,
+  "rebol": null,
+  "registry": null,
+  "remind": null,
+  "resolv": null,
+  "reva": null,
+  "rexx": null,
+  "rhelp": null,
+  "rib": null,
+  "rnc": null,
+  "rnoweb": null,
+  "robots": null,
+  "rpcgen": null,
+  "rpl": null,
+  "rst": null,
+  "rtf": null,
+  "ruby": "Ruby",
+  "samba": null,
+  "sas": null,
+  "sass": "Sass",
+  "sather": null,
+  "scheme": "Scheme",
+  "scilab": null,
+  "screen": null,
+  "scss": "SCSS",
+  "sd": null,
+  "sdc": null,
+  "sdl": null,
+  "sed": null,
+  "sendpr": null,
+  "sensors": null,
+  "services": null,
+  "setserial": null,
+  "sgml": null,
+  "sgmldecl": null,
+  "sgmllnx": null,
+  "sh": null,
+  "sicad": null,
+  "sieve": null,
+  "simula": null,
+  "sinda": null,
+  "sindacmp": null,
+  "sindaout": null,
+  "sisu": null,
+  "skill": "SKILL",
+  "sl": null,
+  "slang": null,
+  "slice": null,
+  "slpconf": null,
+  "slpreg": null,
+  "slpspi": null,
+  "slrnrc": null,
+  "slrnsc": null,
+  "sm": null,
+  "smarty": null,
+  "smcl": null,
+  "smil": null,
+  "smith": null,
+  "sml": null,
+  "snnsnet": null,
+  "snnspat": null,
+  "snnsres": null,
+  "snobol4": null,
+  "spec": null,
+  "specman": null,
+  "spice": null,
+  "splint": null,
+  "spup": null,
+  "spyce": null,
+  "sql": null,
+  "sqlanywhere": null,
+  "sqlforms": null,
+  "sqlinformix": null,
+  "sqlj": null,
+  "sqloracle": null,
+  "sqr": null,
+  "squid": null,
+  "sshconfig": null,
+  "sshdconfig": null,
+  "st": null,
+  "stata": null,
+  "stp": null,
+  "strace": null,
+  "sudoers": null,
+  "svg": null,
+  "svn": null,
+  "syncolor": null,
+  "synload": null,
+  "syntax": null,
+  "sysctl": null,
+  "tads": null,
+  "tags": null,
+  "tak": null,
+  "takcmp": null,
+  "takout": null,
+  "tar": null,
+  "taskdata": null,
+  "taskedit": null,
+  "tasm": null,
+  "tcl": null,
+  "tcsh": null,
+  "terminfo": null,
+  "tex": null,
+  "texinfo": null,
+  "texmf": null,
+  "tf": null,
+  "tidy": null,
+  "tilde": null,
+  "tli": null,
+  "tpp": null,
+  "trasys": null,
+  "trustees": null,
+  "tsalt": null,
+  "tsscl": null,
+  "tssgm": null,
+  "tssop": null,
+  "uc": null,
+  "udevconf": null,
+  "udevperm": null,
+  "udevrules": null,
+  "uil": null,
+  "updatedb": null,
+  "valgrind": null,
+  "vb": "VB.net",
+  "vera": null,
+  "verilog": null,
+  "verilogams": null,
+  "vgrindefs": null,
+  "vhdl": null,
+  "vim": "VimL",
+  "viminfo": null,
+  "virata": null,
+  "vmasm": null,
+  "voscm": null,
+  "vrml": null,
+  "vsejcl": null,
+  "wdiff": null,
+  "web": null,
+  "webmacro": null,
+  "wget": null,
+  "winbatch": null,
+  "wml": null,
+  "wsh": null,
+  "wsml": null,
+  "wvdial": null,
+  "xbl": null,
+  "xdefaults": null,
+  "xf86conf": null,
+  "xhtml": "HTML",
+  "xinetd": null,
+  "xkb": null,
+  "xmath": null,
+  "xml": "XML",
+  "xmodmap": null,
+  "xpm": null,
+  "xpm2": null,
+  "xquery": null,
+  "xs": null,
+  "xsd": null,
+  "xslt": null,
+  "xxd": null,
+  "yacc": null,
+  "yaml": "YAML",
+  "z8a": null,
+  "zsh": null
+}

packages/wakatime/logger.py

@@ -11,9 +11,10 @@
 
 import logging
 import os
-import sys
+import traceback
 
 from .compat import u
+from .packages.requests.packages import urllib3
 try:
     from collections import OrderedDict  # pragma: nocover
 except ImportError:  # pragma: nocover
@@ -24,26 +25,12 @@ except (ImportError, SyntaxError):  # pragma: nocover
     import json
 
 
-class CustomEncoder(json.JSONEncoder):
-
-    def default(self, obj):
-        if isinstance(obj, bytes):  # pragma: nocover
-            obj = u(obj)
-            return json.dumps(obj)
-        try:  # pragma: nocover
-            encoded = super(CustomEncoder, self).default(obj)
-        except UnicodeDecodeError:  # pragma: nocover
-            obj = u(obj)
-            encoded = super(CustomEncoder, self).default(obj)
-        return encoded
-
-
 class JsonFormatter(logging.Formatter):
 
-    def setup(self, timestamp, isWrite, entity, version, plugin, verbose,
+    def setup(self, timestamp, is_write, entity, version, plugin, verbose,
               warnings=False):
         self.timestamp = timestamp
-        self.isWrite = isWrite
+        self.is_write = is_write
         self.entity = entity
         self.version = version
         self.plugin = plugin
@@ -54,24 +41,25 @@ class JsonFormatter(logging.Formatter):
         data = OrderedDict([
             ('now', self.formatTime(record, self.datefmt)),
         ])
-        data['version'] = self.version
-        data['plugin'] = self.plugin
+        data['version'] = u(self.version)
+        if self.plugin:
+            data['plugin'] = u(self.plugin)
         data['time'] = self.timestamp
         if self.verbose:
-            data['caller'] = record.pathname
+            data['caller'] = u(record.pathname)
             data['lineno'] = record.lineno
-            data['isWrite'] = self.isWrite
-            data['file'] = self.entity
-            if not self.isWrite:
-                del data['isWrite']
+            if self.is_write:
+                data['is_write'] = self.is_write
+            data['file'] = u(self.entity)
         data['level'] = record.levelname
-        data['message'] = record.getMessage() if self.warnings else record.msg
-        if not self.plugin:
-            del data['plugin']
-        return CustomEncoder().encode(data)
+        data['message'] = u(record.getMessage() if self.warnings else record.msg)
+        return json.dumps(data)
 
-    def formatException(self, exc_info):
-        return sys.exec_info[2].format_exc()
+    def traceback(self, lvl=None):
+        logger = logging.getLogger('WakaTime')
+        if not lvl:
+            lvl = logger.getEffectiveLevel()
+        logger.log(lvl, traceback.format_exc())
 
 
 def set_log_level(logger, args):
@@ -82,6 +70,7 @@ def set_log_level(logger, args):
 
 
 def setup_logging(args, version):
+    urllib3.disable_warnings()
     logger = logging.getLogger('WakaTime')
     for handler in logger.handlers:
         logger.removeHandler(handler)
@@ -93,7 +82,7 @@ def setup_logging(args, version):
     formatter = JsonFormatter(datefmt='%Y/%m/%d %H:%M:%S %z')
     formatter.setup(
         timestamp=args.timestamp,
-        isWrite=args.isWrite,
+        is_write=args.is_write,
         entity=args.entity,
         version=version,
         plugin=args.plugin,
@@ -102,10 +91,13 @@ def setup_logging(args, version):
     handler.setFormatter(formatter)
     logger.addHandler(handler)
 
+    # add custom traceback logging method
+    logger.traceback = formatter.traceback
+
     warnings_formatter = JsonFormatter(datefmt='%Y/%m/%d %H:%M:%S %z')
     warnings_formatter.setup(
         timestamp=args.timestamp,
-        isWrite=args.isWrite,
+        is_write=args.is_write,
         entity=args.entity,
         version=version,
         plugin=args.plugin,

packages/wakatime/main.py

@@ -3,7 +3,7 @@
     wakatime.main
     ~~~~~~~~~~~~~
 
-    wakatime module entry point.
+    Module entry point.
 
     :copyright: (c) 2013 Alan Hamlett.
     :license: BSD, see LICENSE for more details.
@@ -14,285 +14,66 @@ from __future__ import print_function
 import base64
 import logging
 import os
-import platform
 import re
 import sys
-import time
 import traceback
 import socket
-try:
-    import ConfigParser as configparser
-except ImportError:  # pragma: nocover
-    import configparser
 
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'packages'))
+pwd = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.dirname(pwd))
+sys.path.insert(0, os.path.join(pwd, 'packages'))
 
 from .__about__ import __version__
-from .compat import u, open, is_py3
+from .arguments import parseArguments
+from .compat import u, is_py3
+from .constants import (
+    API_ERROR,
+    AUTH_ERROR,
+    SUCCESS,
+    UNKNOWN_ERROR,
+    MALFORMED_HEARTBEAT_ERROR,
+)
 from .logger import setup_logging
+
+log = logging.getLogger('WakaTime')
+
+try:
+    from .packages import requests
+except ImportError:
+    log.traceback(logging.ERROR)
+    print(traceback.format_exc())
+    log.error('Please upgrade Python to the latest version.')
+    print('Please upgrade Python to the latest version.')
+    sys.exit(UNKNOWN_ERROR)
+
 from .offlinequeue import Queue
-from .packages import argparse
 from .packages.requests.exceptions import RequestException
 from .project import get_project_info
 from .session_cache import SessionCache
 from .stats import get_file_stats
+from .utils import get_user_agent, should_exclude, format_file_path
 try:
     from .packages import simplejson as json  # pragma: nocover
 except (ImportError, SyntaxError):  # pragma: nocover
     import json
-try:
-    from .packages import tzlocal
-except:  # pragma: nocover
-    from .packages import tzlocal3 as tzlocal
+from .packages import tzlocal
 
 
-log = logging.getLogger('WakaTime')
-
-
-class FileAction(argparse.Action):
-
-    def __call__(self, parser, namespace, values, option_string=None):
-        try:
-            if os.path.isfile(values):
-                values = os.path.realpath(values)
-        except:  # pragma: nocover
-            pass
-        setattr(namespace, self.dest, values)
-
-
-def parseConfigFile(configFile=None):
-    """Returns a configparser.SafeConfigParser instance with configs
-    read from the config file. Default location of the config file is
-    at ~/.wakatime.cfg.
-    """
-
-    if not configFile:
-        configFile = os.path.join(os.path.expanduser('~'), '.wakatime.cfg')
-
-    configs = configparser.SafeConfigParser()
-    try:
-        with open(configFile, 'r', encoding='utf-8') as fh:
-            try:
-                configs.readfp(fh)
-            except configparser.Error:
-                print(traceback.format_exc())
-                return None
-    except IOError:
-        print(u('Error: Could not read from config file {0}').format(u(configFile)))
-    return configs
-
-
-def parseArguments():
-    """Parse command line arguments and configs from ~/.wakatime.cfg.
-    Command line arguments take precedence over config file settings.
-    Returns instances of ArgumentParser and SafeConfigParser.
-    """
-
-    # define supported command line arguments
-    parser = argparse.ArgumentParser(
-            description='Common interface for the WakaTime api.')
-    parser.add_argument('--entity', dest='entity', metavar='FILE',
-            action=FileAction,
-            help='absolute path to file for the heartbeat; can also be a '+
-                 'url, domain, or app when --entitytype is not file')
-    parser.add_argument('--file', dest='file', action=FileAction,
-            help=argparse.SUPPRESS)
-    parser.add_argument('--key', dest='key',
-            help='your wakatime api key; uses api_key from '+
-                '~/.wakatime.conf by default')
-    parser.add_argument('--write', dest='isWrite',
-            action='store_true',
-            help='when set, tells api this heartbeat was triggered from '+
-                 'writing to a file')
-    parser.add_argument('--plugin', dest='plugin',
-            help='optional text editor plugin name and version '+
-                 'for User-Agent header')
-    parser.add_argument('--time', dest='timestamp', metavar='time',
-            type=float,
-            help='optional floating-point unix epoch timestamp; '+
-                 'uses current time by default')
-    parser.add_argument('--lineno', dest='lineno',
-            help='optional line number; current line being edited')
-    parser.add_argument('--cursorpos', dest='cursorpos',
-            help='optional cursor position in the current file')
-    parser.add_argument('--entitytype', dest='entity_type',
-            help='entity type for this heartbeat. can be one of "file", '+
-                 '"url", "domain", or "app"; defaults to file.')
-    parser.add_argument('--proxy', dest='proxy',
-                        help='optional https proxy url; for example: '+
-                        'https://user:pass@localhost:8080')
-    parser.add_argument('--project', dest='project',
-            help='optional project name')
-    parser.add_argument('--alternate-project', dest='alternate_project',
-            help='optional alternate project name; auto-discovered project takes priority')
-    parser.add_argument('--hostname', dest='hostname', help='hostname of current machine.')
-    parser.add_argument('--disableoffline', dest='offline',
-            action='store_false',
-            help='disables offline time logging instead of queuing logged time')
-    parser.add_argument('--hidefilenames', dest='hidefilenames',
-            action='store_true',
-            help='obfuscate file names; will not send file names to api')
-    parser.add_argument('--exclude', dest='exclude', action='append',
-            help='filename patterns to exclude from logging; POSIX regex '+
-                 'syntax; can be used more than once')
-    parser.add_argument('--include', dest='include', action='append',
-            help='filename patterns to log; when used in combination with '+
-                 '--exclude, files matching include will still be logged; '+
-                 'POSIX regex syntax; can be used more than once')
-    parser.add_argument('--ignore', dest='ignore', action='append',
-            help=argparse.SUPPRESS)
-    parser.add_argument('--logfile', dest='logfile',
-            help='defaults to ~/.wakatime.log')
-    parser.add_argument('--apiurl', dest='api_url',
-            help='heartbeats api url; for debugging with a local server')
-    parser.add_argument('--timeout', dest='timeout', type=int,
-            help='number of seconds to wait when sending heartbeats to api')
-    parser.add_argument('--config', dest='config',
-            help='defaults to ~/.wakatime.conf')
-    parser.add_argument('--verbose', dest='verbose', action='store_true',
-            help='turns on debug messages in log file')
-    parser.add_argument('--version', action='version', version=__version__)
-
-    # parse command line arguments
-    args = parser.parse_args()
-
-    # use current unix epoch timestamp by default
-    if not args.timestamp:
-        args.timestamp = time.time()
-
-    # parse ~/.wakatime.cfg file
-    configs = parseConfigFile(args.config)
-    if configs is None:
-        return args, configs
-
-    # update args from configs
-    if not args.key:
-        default_key = None
-        if configs.has_option('settings', 'api_key'):
-            default_key = configs.get('settings', 'api_key')
-        elif configs.has_option('settings', 'apikey'):
-            default_key = configs.get('settings', 'apikey')
-        if default_key:
-            args.key = default_key
-        else:
-            parser.error('Missing api key')
-    if not args.entity_type:
-        args.entity_type = 'file'
-    if not args.entity:
-        if args.file:
-            args.entity = args.file
-        else:
-            parser.error('argument --entity is required')
-    if not args.exclude:
-        args.exclude = []
-    if configs.has_option('settings', 'ignore'):
-        try:
-            for pattern in configs.get('settings', 'ignore').split("\n"):
-                if pattern.strip() != '':
-                    args.exclude.append(pattern)
-        except TypeError:  # pragma: nocover
-            pass
-    if configs.has_option('settings', 'exclude'):
-        try:
-            for pattern in configs.get('settings', 'exclude').split("\n"):
-                if pattern.strip() != '':
-                    args.exclude.append(pattern)
-        except TypeError:  # pragma: nocover
-            pass
-    if not args.include:
-        args.include = []
-    if configs.has_option('settings', 'include'):
-        try:
-            for pattern in configs.get('settings', 'include').split("\n"):
-                if pattern.strip() != '':
-                    args.include.append(pattern)
-        except TypeError:  # pragma: nocover
-            pass
-    if args.offline and configs.has_option('settings', 'offline'):
-        args.offline = configs.getboolean('settings', 'offline')
-    if not args.hidefilenames and configs.has_option('settings', 'hidefilenames'):
-        args.hidefilenames = configs.getboolean('settings', 'hidefilenames')
-    if not args.proxy and configs.has_option('settings', 'proxy'):
-        args.proxy = configs.get('settings', 'proxy')
-    if not args.verbose and configs.has_option('settings', 'verbose'):
-        args.verbose = configs.getboolean('settings', 'verbose')
-    if not args.verbose and configs.has_option('settings', 'debug'):
-        args.verbose = configs.getboolean('settings', 'debug')
-    if not args.logfile and configs.has_option('settings', 'logfile'):
-        args.logfile = configs.get('settings', 'logfile')
-    if not args.api_url and configs.has_option('settings', 'api_url'):
-        args.api_url = configs.get('settings', 'api_url')
-    if not args.timeout and configs.has_option('settings', 'timeout'):
-        try:
-            args.timeout = int(configs.get('settings', 'timeout'))
-        except ValueError:
-            print(traceback.format_exc())
-
-    return args, configs
-
-
-def should_exclude(entity, include, exclude):
-    if entity is not None and entity.strip() != '':
-        try:
-            for pattern in include:
-                try:
-                    compiled = re.compile(pattern, re.IGNORECASE)
-                    if compiled.search(entity):
-                        return False
-                except re.error as ex:
-                    log.warning(u('Regex error ({msg}) for include pattern: {pattern}').format(
-                        msg=u(ex),
-                        pattern=u(pattern),
-                    ))
-        except TypeError:  # pragma: nocover
-            pass
-        try:
-            for pattern in exclude:
-                try:
-                    compiled = re.compile(pattern, re.IGNORECASE)
-                    if compiled.search(entity):
-                        return pattern
-                except re.error as ex:
-                    log.warning(u('Regex error ({msg}) for exclude pattern: {pattern}').format(
-                        msg=u(ex),
-                        pattern=u(pattern),
-                    ))
-        except TypeError:  # pragma: nocover
-            pass
-    return False
-
-
-def get_user_agent(plugin):
-    ver = sys.version_info
-    python_version = '%d.%d.%d.%s.%d' % (ver[0], ver[1], ver[2], ver[3], ver[4])
-    user_agent = u('wakatime/{ver} ({platform}) Python{py_ver}').format(
-        ver=u(__version__),
-        platform=u(platform.platform()),
-        py_ver=python_version,
-    )
-    if plugin:
-        user_agent = u('{user_agent} {plugin}').format(
-            user_agent=user_agent,
-            plugin=u(plugin),
-        )
-    else:
-        user_agent = u('{user_agent} Unknown/0').format(
-            user_agent=user_agent,
-        )
-    return user_agent
-
-
-def send_heartbeat(project=None, branch=None, hostname=None, stats={}, key=None, entity=None,
-        timestamp=None, isWrite=None, plugin=None, offline=None, entity_type='file',
-        hidefilenames=None, proxy=None, api_url=None, timeout=None, **kwargs):
+def send_heartbeat(project=None, branch=None, hostname=None, stats={}, key=None,
+                   entity=None, timestamp=None, is_write=None, plugin=None,
+                   offline=None, entity_type='file', hidefilenames=None,
+                   proxy=None, nosslverify=None, api_url=None, timeout=None,
+                   use_ntlm_proxy=False, **kwargs):
     """Sends heartbeat as POST request to WakaTime api server.
+
+    Returns `SUCCESS` when heartbeat was sent, otherwise returns an
+    error code constant.
     """
 
     if not api_url:
-        api_url = 'https://wakatime.com/api/v1/heartbeats'
+        api_url = 'https://api.wakatime.com/api/v1/heartbeats'
     if not timeout:
-        timeout = 30
+        timeout = 60
     log.debug('Sending heartbeat to api at %s' % api_url)
     data = {
         'time': timestamp,
@@ -300,8 +81,18 @@ def send_heartbeat(project=None, branch=None, hostname=None, stats={}, key=None,
         'type': entity_type,
     }
     if hidefilenames and entity is not None and entity_type == 'file':
-        extension = u(os.path.splitext(data['entity'])[1])
-        data['entity'] = u('HIDDEN{0}').format(extension)
+        for pattern in hidefilenames:
+            try:
+                compiled = re.compile(pattern, re.IGNORECASE)
+                if compiled.search(entity):
+                    extension = u(os.path.splitext(data['entity'])[1])
+                    data['entity'] = u('HIDDEN{0}').format(extension)
+                    break
+            except re.error as ex:
+                log.warning(u('Regex error ({msg}) for include pattern: {pattern}').format(
+                    msg=u(ex),
+                    pattern=u(pattern),
+                ))
     if stats.get('lines'):
         data['lines'] = stats['lines']
     if stats.get('language'):
@@ -312,8 +103,8 @@ def send_heartbeat(project=None, branch=None, hostname=None, stats={}, key=None,
         data['lineno'] = stats['lineno']
     if stats.get('cursorpos'):
         data['cursorpos'] = stats['cursorpos']
-    if isWrite:
-        data['is_write'] = isWrite
+    if is_write:
+        data['is_write'] = is_write
     if project:
         data['project'] = project
     if branch:
@@ -331,10 +122,7 @@ def send_heartbeat(project=None, branch=None, hostname=None, stats={}, key=None,
         'Authorization': auth,
     }
     if hostname:
-        headers['X-Machine-Name'] = hostname
-    proxies = {}
-    if proxy:
-        proxies['https'] = proxy
+        headers['X-Machine-Name'] = u(hostname).encode('utf-8')
 
     # add Olson timezone to request
     try:
@@ -342,64 +130,235 @@ def send_heartbeat(project=None, branch=None, hostname=None, stats={}, key=None,
     except:
         tz = None
     if tz:
-        headers['TimeZone'] = u(tz.zone)
+        headers['TimeZone'] = u(tz.zone).encode('utf-8')
 
     session_cache = SessionCache()
     session = session_cache.get()
 
-    # log time to api
+    should_try_ntlm = False
+    proxies = {}
+    if proxy:
+        if use_ntlm_proxy:
+            from .packages.requests_ntlm import HttpNtlmAuth
+            username = proxy.rsplit(':', 1)
+            password = ''
+            if len(username) == 2:
+                password = username[1]
+            username = username[0]
+            session.auth = HttpNtlmAuth(username, password, session)
+        else:
+            should_try_ntlm = '\\' in proxy
+            proxies['https'] = proxy
+
+    # send request to api
     response = None
     try:
         response = session.post(api_url, data=request_body, headers=headers,
-                                 proxies=proxies, timeout=timeout)
+                                proxies=proxies, timeout=timeout,
+                                verify=not nosslverify)
     except RequestException:
-        exception_data = {
-            sys.exc_info()[0].__name__: u(sys.exc_info()[1]),
-        }
-        if log.isEnabledFor(logging.DEBUG):
-            exception_data['traceback'] = traceback.format_exc()
-        if offline:
-            queue = Queue()
-            queue.push(data, json.dumps(stats), plugin)
-            if log.isEnabledFor(logging.DEBUG):
-                log.warn(exception_data)
+        if should_try_ntlm:
+            return send_heartbeat(
+                project=project,
+                entity=entity,
+                timestamp=timestamp,
+                branch=branch,
+                hostname=hostname,
+                stats=stats,
+                key=key,
+                is_write=is_write,
+                plugin=plugin,
+                offline=offline,
+                hidefilenames=hidefilenames,
+                entity_type=entity_type,
+                proxy=proxy,
+                api_url=api_url,
+                timeout=timeout,
+                use_ntlm_proxy=True,
+            )
         else:
-            log.error(exception_data)
-    else:
-        response_code = response.status_code if response is not None else None
-        response_content = response.text if response is not None else None
-        if response_code == 201:
-            log.debug({
-                'response_code': response_code,
-            })
-            session_cache.save(session)
-            return True
-        if offline:
-            if response_code != 400:
+            exception_data = {
+                sys.exc_info()[0].__name__: u(sys.exc_info()[1]),
+            }
+            if log.isEnabledFor(logging.DEBUG):
+                exception_data['traceback'] = traceback.format_exc()
+            if offline:
                 queue = Queue()
                 queue.push(data, json.dumps(stats), plugin)
-                if response_code == 401:
+                if log.isEnabledFor(logging.DEBUG):
+                    log.warn(exception_data)
+            else:
+                log.error(exception_data)
+
+    except:  # delete cached session when requests raises unknown exception
+        if should_try_ntlm:
+            return send_heartbeat(
+                project=project,
+                entity=entity,
+                timestamp=timestamp,
+                branch=branch,
+                hostname=hostname,
+                stats=stats,
+                key=key,
+                is_write=is_write,
+                plugin=plugin,
+                offline=offline,
+                hidefilenames=hidefilenames,
+                entity_type=entity_type,
+                proxy=proxy,
+                api_url=api_url,
+                timeout=timeout,
+                use_ntlm_proxy=True,
+            )
+        else:
+            exception_data = {
+                sys.exc_info()[0].__name__: u(sys.exc_info()[1]),
+                'traceback': traceback.format_exc(),
+            }
+            if offline:
+                queue = Queue()
+                queue.push(data, json.dumps(stats), plugin)
+                log.warn(exception_data)
+            session_cache.delete()
+            return API_ERROR
+
+    else:
+        code = response.status_code if response is not None else None
+        content = response.text if response is not None else None
+        if code == requests.codes.created or code == requests.codes.accepted:
+            log.debug({
+                'response_code': code,
+            })
+            session_cache.save(session)
+            return SUCCESS
+        if should_try_ntlm:
+            return send_heartbeat(
+                project=project,
+                entity=entity,
+                timestamp=timestamp,
+                branch=branch,
+                hostname=hostname,
+                stats=stats,
+                key=key,
+                is_write=is_write,
+                plugin=plugin,
+                offline=offline,
+                hidefilenames=hidefilenames,
+                entity_type=entity_type,
+                proxy=proxy,
+                api_url=api_url,
+                timeout=timeout,
+                use_ntlm_proxy=True,
+            )
+        else:
+            if offline:
+                if code != 400:
+                    queue = Queue()
+                    queue.push(data, json.dumps(stats), plugin)
+                    if code == 401:
+                        log.error({
+                            'response_code': code,
+                            'response_content': content,
+                        })
+                        session_cache.delete()
+                        return AUTH_ERROR
+                    elif log.isEnabledFor(logging.DEBUG):
+                        log.warn({
+                            'response_code': code,
+                            'response_content': content,
+                        })
+                else:
                     log.error({
-                        'response_code': response_code,
-                        'response_content': response_content,
-                    })
-                elif log.isEnabledFor(logging.DEBUG):
-                    log.warn({
-                        'response_code': response_code,
-                        'response_content': response_content,
+                        'response_code': code,
+                        'response_content': content,
                     })
             else:
                 log.error({
-                    'response_code': response_code,
-                    'response_content': response_content,
+                    'response_code': code,
+                    'response_content': content,
                 })
-        else:
-            log.error({
-                'response_code': response_code,
-                'response_content': response_content,
-            })
     session_cache.delete()
-    return False
+    return API_ERROR
+
+
+def sync_offline_heartbeats(args, hostname):
+    """Sends all heartbeats which were cached in the offline Queue."""
+
+    queue = Queue()
+    while True:
+        heartbeat = queue.pop()
+        if heartbeat is None:
+            break
+        status = send_heartbeat(
+            project=heartbeat['project'],
+            entity=heartbeat['entity'],
+            timestamp=heartbeat['time'],
+            branch=heartbeat['branch'],
+            hostname=hostname,
+            stats=json.loads(heartbeat['stats']),
+            key=args.key,
+            is_write=heartbeat['is_write'],
+            plugin=heartbeat['plugin'],
+            offline=args.offline,
+            hidefilenames=args.hidefilenames,
+            entity_type=heartbeat['type'],
+            proxy=args.proxy,
+            api_url=args.api_url,
+            timeout=args.timeout,
+        )
+        if status != SUCCESS:
+            if status == AUTH_ERROR:
+                return AUTH_ERROR
+            break
+    return SUCCESS
+
+
+def process_heartbeat(args, configs, hostname, heartbeat):
+    exclude = should_exclude(heartbeat['entity'], args.include, args.exclude)
+    if exclude is not False:
+        log.debug(u('Skipping because matches exclude pattern: {pattern}').format(
+            pattern=u(exclude),
+        ))
+        return SUCCESS
+
+    if heartbeat.get('entity_type') not in ['file', 'domain', 'app']:
+        heartbeat['entity_type'] = 'file'
+
+    if heartbeat['entity_type'] == 'file':
+        heartbeat['entity'] = format_file_path(heartbeat['entity'])
+
+    if heartbeat['entity_type'] != 'file' or os.path.isfile(heartbeat['entity']):
+
+        stats = get_file_stats(heartbeat['entity'],
+                                entity_type=heartbeat['entity_type'],
+                                lineno=heartbeat.get('lineno'),
+                                cursorpos=heartbeat.get('cursorpos'),
+                                plugin=args.plugin,
+                                language=heartbeat.get('language'))
+
+        project = heartbeat.get('project') or heartbeat.get('alternate_project')
+        branch = None
+        if heartbeat['entity_type'] == 'file':
+            project, branch = get_project_info(configs, heartbeat)
+
+        heartbeat['project'] = project
+        heartbeat['branch'] = branch
+        heartbeat['stats'] = stats
+        heartbeat['hostname'] = hostname
+        heartbeat['timeout'] = args.timeout
+        heartbeat['key'] = args.key
+        heartbeat['plugin'] = args.plugin
+        heartbeat['offline'] = args.offline
+        heartbeat['hidefilenames'] = args.hidefilenames
+        heartbeat['proxy'] = args.proxy
+        heartbeat['nosslverify'] = args.nosslverify
+        heartbeat['api_url'] = args.api_url
+
+        return send_heartbeat(**heartbeat)
+
+    else:
+        log.debug('File does not exist; ignoring this heartbeat.')
+        return SUCCESS
 
 
 def execute(argv=None):
@@ -407,64 +366,29 @@ def execute(argv=None):
         sys.argv = ['wakatime'] + argv
 
     args, configs = parseArguments()
-    if configs is None:
-        return 103 # config file parsing error
 
     setup_logging(args, __version__)
 
-    exclude = should_exclude(args.entity, args.include, args.exclude)
-    if exclude is not False:
-        log.debug(u('Skipping because matches exclude pattern: {pattern}').format(
-            pattern=u(exclude),
-        ))
-        return 0
+    try:
 
-    if args.entity_type != 'file' or os.path.isfile(args.entity):
+        hostname = args.hostname or socket.gethostname()
 
-        stats = get_file_stats(args.entity, entity_type=args.entity_type,
-                               lineno=args.lineno, cursorpos=args.cursorpos)
+        heartbeat = vars(args)
+        retval = process_heartbeat(args, configs, hostname, heartbeat)
 
-        project = args.project or args.alternate_project
-        branch = None
-        if args.entity_type == 'file':
-            project, branch = get_project_info(configs, args)
+        if args.extra_heartbeats:
+            try:
+                for heartbeat in json.loads(sys.stdin.readline()):
+                    retval = process_heartbeat(args, configs, hostname, heartbeat)
+            except json.JSONDecodeError:
+                retval = MALFORMED_HEARTBEAT_ERROR
 
-        kwargs = vars(args)
-        kwargs['project'] = project
-        kwargs['branch'] = branch
-        kwargs['stats'] = stats
-        kwargs['hostname'] = args.hostname or socket.gethostname()
-        kwargs['timeout'] = args.timeout
+        if retval == SUCCESS:
+            retval = sync_offline_heartbeats(args, hostname)
 
-        if send_heartbeat(**kwargs):
-            queue = Queue()
-            while True:
-                heartbeat = queue.pop()
-                if heartbeat is None:
-                    break
-                sent = send_heartbeat(
-                    project=heartbeat['project'],
-                    entity=heartbeat['entity'],
-                    timestamp=heartbeat['time'],
-                    branch=heartbeat['branch'],
-                    hostname=kwargs['hostname'],
-                    stats=json.loads(heartbeat['stats']),
-                    key=args.key,
-                    isWrite=heartbeat['is_write'],
-                    plugin=heartbeat['plugin'],
-                    offline=args.offline,
-                    hidefilenames=args.hidefilenames,
-                    entity_type=heartbeat['type'],
-                    proxy=args.proxy,
-                    api_url=args.api_url,
-                    timeout=args.timeout,
-                )
-                if not sent:
-                    break
-            return 0 # success
+        return retval
 
-        return 102 # api error
-
-    else:
-        log.debug('File does not exist; ignoring this heartbeat.')
-        return 0
+    except:
+        log.traceback(logging.ERROR)
+        print(traceback.format_exc())
+        return UNKNOWN_ERROR

packages/wakatime/offlinequeue.py

@@ -12,7 +12,6 @@
 
 import logging
 import os
-import traceback
 from time import sleep
 
 try:
@@ -35,7 +34,7 @@ class Queue(object):
         return self.db_file
 
     def connect(self):
-        conn = sqlite3.connect(self.get_db_file())
+        conn = sqlite3.connect(self.get_db_file(), isolation_level=None)
         c = conn.cursor()
         c.execute('''CREATE TABLE IF NOT EXISTS {0} (
             entity text,
@@ -70,7 +69,7 @@ class Queue(object):
             conn.commit()
             conn.close()
         except sqlite3.Error:
-            log.error(traceback.format_exc())
+            log.traceback()
 
     def pop(self):
         if not HAS_SQL:  # pragma: nocover
@@ -81,7 +80,7 @@ class Queue(object):
         try:
             conn, c = self.connect()
         except sqlite3.Error:
-            log.debug(traceback.format_exc())
+            log.traceback(logging.DEBUG)
             return None
         loop = True
         while loop and tries > -1:
@@ -119,11 +118,11 @@ class Queue(object):
                     }
                 loop = False
             except sqlite3.Error:  # pragma: nocover
-                log.debug(traceback.format_exc())
+                log.traceback(logging.DEBUG)
                 sleep(wait)
                 tries -= 1
         try:
             conn.close()
         except sqlite3.Error:  # pragma: nocover
-            log.debug(traceback.format_exc())
+            log.traceback(logging.DEBUG)
         return heartbeat

packages/wakatime/packages/configparser/helpers.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from collections import MutableMapping
+try:
+    from collections import UserDict
+except ImportError:
+    from UserDict import UserDict
+
+try:
+    from collections import OrderedDict
+except ImportError:
+    from ordereddict import OrderedDict
+
+from io import open
+import sys
+try:
+    from thread import get_ident
+except ImportError:
+    try:
+        from _thread import get_ident
+    except ImportError:
+        from _dummy_thread import get_ident
+
+
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+
+str = type('str')
+
+
+def from_none(exc):
+    """raise from_none(ValueError('a')) == raise ValueError('a') from None"""
+    exc.__cause__ = None
+    exc.__suppress_context__ = True
+    return exc
+
+
+# from reprlib 3.2.1
+def recursive_repr(fillvalue='...'):
+    'Decorator to make a repr function return fillvalue for a recursive call'
+
+    def decorating_function(user_function):
+        repr_running = set()
+
+        def wrapper(self):
+            key = id(self), get_ident()
+            if key in repr_running:
+                return fillvalue
+            repr_running.add(key)
+            try:
+                result = user_function(self)
+            finally:
+                repr_running.discard(key)
+            return result
+
+        # Can't use functools.wraps() here because of bootstrap issues
+        wrapper.__module__ = getattr(user_function, '__module__')
+        wrapper.__doc__ = getattr(user_function, '__doc__')
+        wrapper.__name__ = getattr(user_function, '__name__')
+        wrapper.__annotations__ = getattr(user_function, '__annotations__', {})
+        return wrapper
+
+    return decorating_function
+
+# from collections 3.2.1
+class _ChainMap(MutableMapping):
+    ''' A ChainMap groups multiple dicts (or other mappings) together
+    to create a single, updateable view.
+
+    The underlying mappings are stored in a list.  That list is public and can
+    accessed or updated using the *maps* attribute.  There is no other state.
+
+    Lookups search the underlying mappings successively until a key is found.
+    In contrast, writes, updates, and deletions only operate on the first
+    mapping.
+
+    '''
+
+    def __init__(self, *maps):
+        '''Initialize a ChainMap by setting *maps* to the given mappings.
+        If no mappings are provided, a single empty dictionary is used.
+
+        '''
+        self.maps = list(maps) or [{}]          # always at least one map
+
+    def __missing__(self, key):
+        raise KeyError(key)
+
+    def __getitem__(self, key):
+        for mapping in self.maps:
+            try:
+                return mapping[key]             # can't use 'key in mapping' with defaultdict
+            except KeyError:
+                pass
+        return self.__missing__(key)            # support subclasses that define __missing__
+
+    def get(self, key, default=None):
+        return self[key] if key in self else default
+
+    def __len__(self):
+        return len(set().union(*self.maps))     # reuses stored hash values if possible
+
+    def __iter__(self):
+        return iter(set().union(*self.maps))
+
+    def __contains__(self, key):
+        return any(key in m for m in self.maps)
+
+    @recursive_repr()
+    def __repr__(self):
+        return '{0.__class__.__name__}({1})'.format(
+            self, ', '.join(map(repr, self.maps)))
+
+    @classmethod
+    def fromkeys(cls, iterable, *args):
+        'Create a ChainMap with a single dict created from the iterable.'
+        return cls(dict.fromkeys(iterable, *args))
+
+    def copy(self):
+        'New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]'
+        return self.__class__(self.maps[0].copy(), *self.maps[1:])
+
+    __copy__ = copy
+
+    def new_child(self):                        # like Django's Context.push()
+        'New ChainMap with a new dict followed by all previous maps.'
+        return self.__class__({}, *self.maps)
+
+    @property
+    def parents(self):                          # like Django's Context.pop()
+        'New ChainMap from maps[1:].'
+        return self.__class__(*self.maps[1:])
+
+    def __setitem__(self, key, value):
+        self.maps[0][key] = value
+
+    def __delitem__(self, key):
+        try:
+            del self.maps[0][key]
+        except KeyError:
+            raise KeyError('Key not found in the first mapping: {!r}'.format(key))
+
+    def popitem(self):
+        'Remove and return an item pair from maps[0]. Raise KeyError is maps[0] is empty.'
+        try:
+            return self.maps[0].popitem()
+        except KeyError:
+            raise KeyError('No keys found in the first mapping.')
+
+    def pop(self, key, *args):
+        'Remove *key* from maps[0] and return its value. Raise KeyError if *key* not in maps[0].'
+        try:
+            return self.maps[0].pop(key, *args)
+        except KeyError:
+            raise KeyError('Key not found in the first mapping: {!r}'.format(key))
+
+    def clear(self):
+        'Clear maps[0], leaving maps[1:] intact.'
+        self.maps[0].clear()
+
+
+try:
+    from collections import ChainMap
+except ImportError:
+    ChainMap = _ChainMap

packages/wakatime/packages/ntlm_auth/U32.py

@@ -0,0 +1,156 @@
+# This file is part of 'NTLM Authorization Proxy Server' http://sourceforge.net/projects/ntlmaps/
+# Copyright 2001 Dmitry A. Rozmanov <dima@xenon.spb.ru>
+#
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+from __future__ import division
+import six
+
+C = 0x1000000000
+
+
+def norm(n):
+    return n & 0xFFFFFFFF
+
+
+class U32:
+    v = 0
+
+    def __init__(self, value=0):
+        if not isinstance(value, six.integer_types):
+            value = six.byte2int(value)
+
+        self.v = C + norm(abs(int(value)))
+
+    def set(self, value=0):
+        self.v = C + norm(abs(int(value)))
+
+    def __repr__(self):
+        return hex(norm(self.v))
+
+    def __long__(self):
+        return int(norm(self.v))
+
+    def __int__(self):
+        return int(norm(self.v))
+
+    def __chr__(self):
+        return chr(norm(self.v))
+
+    def __add__(self, b):
+        r = U32()
+        r.v = C + norm(self.v + b.v)
+        return r
+
+    def __sub__(self, b):
+        r = U32()
+        if self.v < b.v:
+            r.v = C + norm(0x100000000 - (b.v - self.v))
+        else:
+            r.v = C + norm(self.v - b.v)
+        return r
+
+    def __mul__(self, b):
+        r = U32()
+        r.v = C + norm(self.v * b.v)
+        return r
+
+    def __div__(self, b):
+        r = U32()
+        r.v = C + (norm(self.v) // norm(b.v))
+        return r
+
+    def __truediv__(self, b):
+        r = U32()
+        r.v = C + (norm(self.v) / norm(b.v))
+        return r
+
+    def __mod__(self, b):
+        r = U32()
+        r.v = C + (norm(self.v) % norm(b.v))
+        return r
+
+    def __neg__(self):
+        return U32(self.v)
+
+    def __pos__(self):
+        return U32(self.v)
+
+    def __abs__(self):
+        return U32(self.v)
+
+    def __invert__(self):
+        r = U32()
+        r.v = C + norm(~self.v)
+        return r
+
+    def __lshift__(self, b):
+        r = U32()
+        r.v = C + norm(self.v << b)
+        return r
+
+    def __rshift__(self, b):
+        r = U32()
+        r.v = C + (norm(self.v) >> b)
+        return r
+
+    def __and__(self, b):
+        r = U32()
+        r.v = C + norm(self.v & b.v)
+        return r
+
+    def __or__(self, b):
+        r = U32()
+        r.v = C + norm(self.v | b.v)
+        return r
+
+    def __xor__(self, b):
+        r = U32()
+        r.v = C + norm(self.v ^ b.v)
+        return r
+
+    def __not__(self):
+        return U32(not norm(self.v))
+
+    def truth(self):
+        return norm(self.v)
+
+    def __cmp__(self, b):
+        if norm(self.v) > norm(b.v):
+            return 1
+        elif norm(self.v) < norm(b.v):
+            return -1
+        else:
+            return 0
+
+    def __lt__(self, other):
+        return self.v < other.v
+
+    def __gt__(self, other):
+        return self.v > other.v
+
+    def __eq__(self, other):
+        return self.v == other.v
+
+    def __le__(self, other):
+        return self.v <= other.v
+
+    def __ge__(self, other):
+        return self.v >= other.v
+
+    def __ne__(self, other):
+        return self.v != other.v
+
+    def __nonzero__(self):
+        return norm(self.v)

packages/wakatime/packages/ntlm_auth/__init__.py

@@ -0,0 +1,3 @@
+from . import ntlm, session_security
+
+__all__ = ('ntlm', 'session_security')

packages/wakatime/packages/ntlm_auth/compute_hash.py

@@ -0,0 +1,80 @@
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+import binascii
+import hashlib
+import hmac
+import re
+from ntlm_auth import des
+
+
+def _lmowfv1(password):
+    """
+    [MS-NLMP] v28.0 2016-07-14
+
+    3.3.1 NTLM v1 Authentication
+    Same function as LMOWFv1 in document to create a one way hash of the password. Only
+    used in NTLMv1 auth without session security
+
+    :param password: The password of the user we are trying to authenticate with
+    :return res: A Lan Manager hash of the password supplied
+    """
+
+    # fix the password length to 14 bytes
+    password = password.upper()
+    lm_pw = password[0:14]
+
+    # do hash
+    magic_str = b"KGS!@#$%"  # page 56 in [MS-NLMP v28.0]
+
+    res = b''
+    dobj = des.DES(lm_pw[0:7])
+    res = res + dobj.encrypt(magic_str)
+
+    dobj = des.DES(lm_pw[7:14])
+    res = res + dobj.encrypt(magic_str)
+
+    return res
+
+def _ntowfv1(password):
+    """
+    [MS-NLMP] v28.0 2016-07-14
+
+    3.3.1 NTLM v1 Authentication
+    Same function as NTOWFv1 in document to create a one way hash of the password. Only
+    used in NTLMv1 auth without session security
+
+    :param password: The password of the user we are trying to authenticate with
+    :return digest: An NT hash of the password supplied
+    """
+
+    digest = hashlib.new('md4', password.encode('utf-16le')).digest()
+    return digest
+
+def _ntowfv2(user_name, password, domain_name):
+    """
+    [MS-NLMP] v28.0 2016-07-14
+
+    3.3.2 NTLM v2 Authentication
+    Same function as NTOWFv2 (and LMOWFv2) in document to create a one way hash of the password.
+    This combines some extra security features over the v1 calculations used in NTLMv2 auth.
+
+    :param user_name: The user name of the user we are trying to authenticate with
+    :param password: The password of the user we are trying to authenticate with
+    :param domain_name: The domain name of the user account we are authenticated with
+    :return digest: An NT hash of the parameters supplied
+    """
+    digest = _ntowfv1(password)
+    digest = hmac.new(digest, (user_name.upper() + domain_name).encode('utf-16le')).digest()
+
+    return digest

packages/wakatime/packages/ntlm_auth/compute_keys.py

@@ -0,0 +1,138 @@
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+import binascii
+import hashlib
+import hmac
+from ntlm_auth import des
+from ntlm_auth.constants import NegotiateFlags
+
+def _get_exchange_key_ntlm_v1(negotiate_flags, session_base_key, server_challenge, lm_challenge_response, lm_hash):
+    """
+    [MS-NLMP] v28.0 2016-07-14
+
+    4.3.5.1 KXKEY
+    Calculates the Key Exchange Key for NTLMv1 authentication. Used for signing and sealing messages
+
+    @param negotiate_flags:
+    @param session_base_key: A session key calculated from the user password challenge
+    @param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
+    @param lm_challenge_response: The LmChallengeResponse value computed in ComputeResponse
+    @param lm_hash: The LMOWF computed in Compute Response
+    @return key_exchange_key: The Key Exchange Key (KXKEY) used to sign and seal messages and compute the ExportedSessionKey
+    """
+    if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY:
+        key_exchange_key = hmac.new(session_base_key, server_challenge + lm_challenge_response[:8]).digest()
+    elif negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_LM_KEY:
+        des_handler = des.DES(lm_hash[:7])
+        first_des = des_handler.encrypt(lm_challenge_response[:8])
+        des_handler = des.DES(lm_hash[7:8] + binascii.unhexlify('bdbdbdbdbdbdbd'))
+        second_des = des_handler.encrypt(lm_challenge_response[:8])
+
+        key_exchange_key = first_des + second_des
+    elif negotiate_flags & NegotiateFlags.NTLMSSP_REQUEST_NON_NT_SESSION_KEY:
+        key_exchange_key = lm_hash[:8] + b'\0' * 8
+    else:
+        key_exchange_key = session_base_key
+
+    return key_exchange_key
+
+def _get_exchange_key_ntlm_v2(session_base_key):
+    """
+    [MS-NLMP] v28.0 2016-07-14
+
+    4.3.5.1 KXKEY
+    Calculates the Key Exchange Key for NTLMv2 authentication. Used for signing and sealing messages.
+    According to docs, 'If NTLM v2 is used, KeyExchangeKey MUST be set to the given 128-bit SessionBaseKey
+
+    @param session_base_key: A session key calculated from the user password challenge
+    @return key_exchange_key: The Key Exchange Key (KXKEY) used to sign and seal messages
+    """
+    return session_base_key
+
+def get_sign_key(exported_session_key, magic_constant):
+    """
+    3.4.5.2 SIGNKEY
+
+    @param exported_session_key: A 128-bit session key used to derive signing and sealing keys
+    @param magic_constant: A constant value set in the MS-NLMP documentation (constants.SignSealConstants)
+    @return sign_key: Key used to sign messages
+    """
+
+    sign_key = hashlib.md5(exported_session_key + magic_constant).digest()
+
+    return sign_key
+
+def get_seal_key(negotiate_flags, exported_session_key, magic_constant):
+    """
+    3.4.5.3. SEALKEY
+    Main method to use to calculate the seal_key used to seal (encrypt) messages. This will determine
+    the correct method below to use based on the compatibility flags set and should be called instead
+    of the others
+
+    @param exported_session_key: A 128-bit session key used to derive signing and sealing keys
+    @param negotiate_flags: The negotiate_flags structure sent by the server
+    @param magic_constant: A constant value set in the MS-NLMP documentation (constants.SignSealConstants)
+    @return seal_key: Key used to seal messages
+    """
+
+    if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY:
+        seal_key = _get_seal_key_ntlm2(negotiate_flags, exported_session_key, magic_constant)
+    elif negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_LM_KEY:
+        seal_key = _get_seal_key_ntlm1(negotiate_flags, exported_session_key)
+    else:
+        seal_key = exported_session_key
+
+    return seal_key
+
+def _get_seal_key_ntlm1(negotiate_flags, exported_session_key):
+    """
+    3.4.5.3 SEALKEY
+    Calculates the seal_key used to seal (encrypt) messages. This for authentication where
+    NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY has not been negotiated. Will weaken the keys
+    if NTLMSSP_NEGOTIATE_56 is not negotiated it will default to the 40-bit key
+
+    @param negotiate_flags: The negotiate_flags structure sent by the server
+    @param exported_session_key: A 128-bit session key used to derive signing and sealing keys
+    @return seal_key: Key used to seal messages
+    """
+    if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_56:
+        seal_key = exported_session_key[:7] + binascii.unhexlify('a0')
+    else:
+        seal_key = exported_session_key[:5] + binascii.unhexlify('e538b0')
+
+    return seal_key
+
+def _get_seal_key_ntlm2(negotiate_flags, exported_session_key, magic_constant):
+    """
+    3.4.5.3 SEALKEY
+    Calculates the seal_key used to seal (encrypt) messages. This for authentication where
+    NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY has been negotiated. Will weaken the keys
+    if NTLMSSP_NEGOTIATE_128 is not negotiated, will try NEGOTIATE_56 and then will default
+    to the 40-bit key
+
+    @param negotiate_flags: The negotiate_flags structure sent by the server
+    @param exported_session_key: A 128-bit session key used to derive signing and sealing keys
+    @param magic_constant: A constant value set in the MS-NLMP documentation (constants.SignSealConstants)
+    @return seal_key: Key used to seal messages
+    """
+    if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_128:
+        seal_key = exported_session_key
+    elif negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_56:
+        seal_key = exported_session_key[:7]
+    else:
+        seal_key = exported_session_key[:5]
+
+    seal_key = hashlib.md5(seal_key + magic_constant).digest()
+
+    return seal_key

packages/wakatime/packages/ntlm_auth/compute_response.py

@@ -0,0 +1,397 @@
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+import base64
+import calendar
+import hashlib
+import hmac
+import os
+import struct
+import time
+import ntlm_auth.compute_hash as comphash
+import ntlm_auth.compute_keys as compkeys
+from ntlm_auth import des
+from ntlm_auth.constants import NegotiateFlags, AvFlags
+from ntlm_auth.gss_channel_bindings import GssChannelBindingsStruct
+from ntlm_auth.target_info import TargetInfo
+
+class ComputeResponse():
+    """
+        Constructor for the response computations. This class will compute the various
+        nt and lm challenge responses.
+
+        :param user_name: The user name of the user we are trying to authenticate with
+        :param password: The password of the user we are trying to authenticate with
+        :param domain_name: The domain name of the user account we are authenticated with, default is None
+        :param challenge_message: A ChallengeMessage object that was received from the server after the negotiate_message
+        :param ntlm_compatibility: The Lan Manager Compatibility Level, used to determine what NTLM auth version to use, see Ntlm in ntlm.py for more details
+    """
+    def __init__(self, user_name, password, domain_name, challenge_message, ntlm_compatibility):
+        self._user_name = user_name
+        self._password = password
+        self._domain_name = domain_name
+        self._challenge_message = challenge_message
+        self._negotiate_flags = challenge_message.negotiate_flags
+        self._server_challenge = challenge_message.server_challenge
+        self._server_target_info = challenge_message.target_info
+        self._ntlm_compatibility = ntlm_compatibility
+        self._client_challenge = os.urandom(8)
+
+    def get_lm_challenge_response(self):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        3.3.1 - NTLM v1 Authentication
+        3.3.2 - NTLM v2 Authentication
+
+        This method returns the LmChallengeResponse key based on the ntlm_compatibility chosen
+        and the target_info supplied by the CHALLENGE_MESSAGE. It is quite different from what
+        is set in the document as it combines the NTLMv1, NTLM2 and NTLMv2 methods into one
+        and calls separate methods based on the ntlm_compatibility flag chosen.
+
+        :return: response (LmChallengeResponse) - The LM response to the server challenge. Computed by the client
+        """
+        if self._negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY and self._ntlm_compatibility < 3:
+            response = ComputeResponse._get_LMv1_with_session_security_response(self._client_challenge)
+
+        elif 0 <= self._ntlm_compatibility <= 1:
+            response = ComputeResponse._get_LMv1_response(self._password, self._server_challenge)
+        elif self._ntlm_compatibility == 2:
+            # Based on the compatibility level we don't want to use LM responses, ignore the session_base_key as it is returned in nt
+            response, ignore_key = ComputeResponse._get_NTLMv1_response(self._password, self._server_challenge)
+        else:
+            """
+            [MS-NLMP] v28.0 page 45 - 2016-07-14
+
+            3.1.5.12 Client Received a CHALLENGE_MESSAGE from the Server
+            If NTLMv2 authentication is used and the CHALLENGE_MESSAGE TargetInfo field has an MsvAvTimestamp present,
+            the client SHOULD NOT send the LmChallengeResponse and SHOULD send Z(24) instead.
+            """
+
+            response = ComputeResponse._get_LMv2_response(self._user_name, self._password, self._domain_name,
+                                                          self._server_challenge,
+                                                          self._client_challenge)
+            if self._server_target_info is not None:
+                timestamp = self._server_target_info[TargetInfo.MSV_AV_TIMESTAMP]
+                if timestamp is not None:
+                    response = b'\0' * 24
+
+        return response
+
+    def get_nt_challenge_response(self, lm_challenge_response, server_certificate_hash):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        3.3.1 - NTLM v1 Authentication
+        3.3.2 - NTLM v2 Authentication
+
+        This method returns the NtChallengeResponse key based on the ntlm_compatibility chosen
+        and the target_info supplied by the CHALLENGE_MESSAGE. It is quite different from what
+        is set in the document as it combines the NTLMv1, NTLM2 and NTLMv2 methods into one
+        and calls separate methods based on the ntlm_compatibility value chosen.
+
+        :param lm_challenge_response: The LmChallengeResponse calculated beforeand, used to get the key_exchange_key value
+        :param server_certificate_hash: The SHA256 hash of the server certificate (DER encoded) NTLM is authenticated to.
+                                        Used in Channel Binding Tokens if present, default value is None. See
+                                        AuthenticateMessage in messages.py for more details
+        :return response: (NtChallengeResponse) - The NT response to the server challenge. Computed by the client
+        :return session_base_key: (SessionBaseKey) - A session key calculated from the user password challenge
+        :return target_info: (AV_PAIR) - The AV_PAIR structure used in the nt_challenge calculations
+        """
+        if self._negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY and self._ntlm_compatibility < 3:
+            # The compatibility level is less than 3 which means it doesn't support NTLMv2 but we want extended security so use NTLM2 which is different from NTLMv2
+            # [MS-NLMP] - 3.3.1 NTLMv1 Authentication
+            response, session_base_key = ComputeResponse._get_NTLM2_response(self._password, self._server_challenge, self._client_challenge)
+            key_exchange_key = compkeys._get_exchange_key_ntlm_v1(self._negotiate_flags, session_base_key,
+                                                                  self._server_challenge, lm_challenge_response,
+                                                                  comphash._lmowfv1(self._password))
+            target_info = None
+
+        elif 0 <= self._ntlm_compatibility < 3:
+            response, session_base_key = ComputeResponse._get_NTLMv1_response(self._password, self._server_challenge)
+            key_exchange_key = compkeys._get_exchange_key_ntlm_v1(self._negotiate_flags, session_base_key,
+                                                                  self._server_challenge, lm_challenge_response,
+                                                                  comphash._lmowfv1(self._password))
+            target_info = None
+        else:
+            if self._server_target_info is None:
+                target_info = TargetInfo()
+            else:
+                target_info = self._server_target_info
+
+            if target_info[TargetInfo.MSV_AV_TIMESTAMP] is None:
+                timestamp = get_windows_timestamp()
+            else:
+                timestamp = target_info[TargetInfo.MSV_AV_TIMESTAMP][1]
+
+                # [MS-NLMP] If the CHALLENGE_MESSAGE TargetInfo field has an MsvAvTimestamp present, the client SHOULD provide a MIC
+                target_info[TargetInfo.MSV_AV_FLAGS] = struct.pack("<L", AvFlags.MIC_PROVIDED)
+
+            if server_certificate_hash != None:
+                channel_bindings_hash = ComputeResponse._get_channel_bindings_value(server_certificate_hash)
+                target_info[TargetInfo.MSV_AV_CHANNEL_BINDINGS] = channel_bindings_hash
+
+            response, session_base_key = ComputeResponse._get_NTLMv2_response(self._user_name, self._password, self._domain_name,
+                                                 self._server_challenge, self._client_challenge, timestamp, target_info)
+
+            key_exchange_key = compkeys._get_exchange_key_ntlm_v2(session_base_key)
+
+        return response, key_exchange_key, target_info
+
+
+    @staticmethod
+    def _get_LMv1_response(password, server_challenge):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.2.3 LM_RESPONSE
+        The LM_RESPONSE structure defines the NTLM v1 authentication LmChallengeResponse
+        in the AUTHENTICATE_MESSAGE. This response is used only when NTLM v1
+        authentication is configured.
+
+        :param password: The password of the user we are trying to authenticate with
+        :param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
+        :return response: LmChallengeResponse to the server challenge
+        """
+        lm_hash = comphash._lmowfv1(password)
+        response = ComputeResponse._calc_resp(lm_hash, server_challenge)
+
+        return response
+
+    @staticmethod
+    def _get_LMv1_with_session_security_response(client_challenge):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.2.3 LM_RESPONSE
+        The LM_RESPONSE structure defines the NTLM v1 authentication LmChallengeResponse
+        in the AUTHENTICATE_MESSAGE. This response is used only when NTLM v1
+        authentication is configured and NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY is flages.
+
+        :param client_challenge: A random 8-byte response generated by the client for the AUTHENTICATE_MESSAGE
+        :return response: LmChallengeResponse to the server challenge
+        """
+
+        response = client_challenge + b'\0' * 16
+
+        return response
+
+    @staticmethod
+    def _get_LMv2_response(user_name, password, domain_name, server_challenge, client_challenge):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.2.4 LMv2_RESPONSE
+        The LMv2_RESPONSE structure defines the NTLM v2 authentication LmChallengeResponse
+        in the AUTHENTICATE_MESSAGE. This response is used only when NTLM v2
+        authentication is configured.
+
+        :param user_name: The user name of the user we are trying to authenticate with
+        :param password: The password of the user we are trying to authenticate with
+        :param domain_name: The domain name of the user account we are authenticated with
+        :param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
+        :param client_challenge: A random 8-byte response generated by the client for the AUTHENTICATE_MESSAGE
+        :return response: LmChallengeResponse to the server challenge
+        """
+        nt_hash = comphash._ntowfv2(user_name, password, domain_name)
+        lm_hash = hmac.new(nt_hash, (server_challenge + client_challenge)).digest()
+        response = lm_hash + client_challenge
+
+        return response
+
+    @staticmethod
+    def _get_NTLMv1_response(password, server_challenge):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.2.6 NTLM v1 Response: NTLM_RESPONSE
+        The NTLM_RESPONSE strucutre defines the NTLM v1 authentication NtChallengeResponse
+        in the AUTHENTICATE_MESSAGE. This response is only used when NTLM v1 authentication
+        is configured.
+
+        :param password: The password of the user we are trying to authenticate with
+        :param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
+        :return response: NtChallengeResponse to the server_challenge
+        :return session_base_key: A session key calculated from the user password challenge
+        """
+        ntlm_hash = comphash._ntowfv1(password)
+        response = ComputeResponse._calc_resp(ntlm_hash, server_challenge)
+
+        session_base_key = hashlib.new('md4', ntlm_hash).digest()
+
+        return response, session_base_key
+
+    @staticmethod
+    def _get_NTLM2_response(password, server_challenge, client_challenge):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        This name is really misleading as it isn't NTLM v2 authentication rather
+        This authentication is only used when the ntlm_compatibility level is set
+        to a value < 3 (No NTLMv2 auth) but the NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY
+        flag is set in the negotiate flags section. The documentation for computing this
+        value is on page 56 under section 3.3.1 NTLM v1 Authentication
+
+        :param password: The password of the user we are trying to authenticate with
+        :param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
+        :param client_challenge: A random 8-byte response generated by the client for the AUTHENTICATE_MESSAGE
+        :return response: NtChallengeResponse to the server_challenge
+        :return session_base_key: A session key calculated from the user password challenge
+        """
+        ntlm_hash = comphash._ntowfv1(password)
+        nt_session_hash = hashlib.md5(server_challenge + client_challenge).digest()[:8]
+        response = ComputeResponse._calc_resp(ntlm_hash, nt_session_hash[0:8])
+
+        session_base_key = hashlib.new('md4', ntlm_hash).digest()
+
+        return response, session_base_key
+
+    @staticmethod
+    def _get_NTLMv2_response(user_name, password, domain_name, server_challenge, client_challenge, timestamp, target_info):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.2.8 NTLM V2 Response: NTLMv2_RESPONSE
+        The NTLMv2_RESPONSE strucutre defines the NTLMv2 authentication NtChallengeResponse
+        in the AUTHENTICATE_MESSAGE. This response is used only when NTLMv2 authentication
+        is configured.
+
+        The guide on how this is computed is in 3.3.2 NTLM v2 Authentication.
+
+        :param user_name: The user name of the user we are trying to authenticate with
+        :param password: The password of the user we are trying to authenticate with
+        :param domain_name: The domain name of the user account we are authenticated with
+        :param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
+        :param client_challenge: A random 8-byte response generated by the client for the AUTHENTICATE_MESSAGE
+        :param timestamp: An 8-byte timestamp in windows format, 100 nanoseconds since 1601-01-01
+        :param target_info: The target_info structure from the CHALLENGE_MESSAGE with the CBT attached if required
+        :return response: NtChallengeResponse to the server_challenge
+        :return session_base_key: A session key calculated from the user password challenge
+        """
+
+        nt_hash = comphash._ntowfv2(user_name, password, domain_name)
+        temp = ComputeResponse._get_NTLMv2_temp(timestamp, client_challenge, target_info)
+        nt_proof_str = hmac.new(nt_hash, (server_challenge + temp)).digest()
+        response = nt_proof_str + temp
+
+        session_base_key = hmac.new(nt_hash, nt_proof_str).digest()
+
+        return response, session_base_key
+
+    @staticmethod
+    def _get_NTLMv2_temp(timestamp, client_challenge, target_info):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.2.7 NTLMv2_CLIENT_CHALLENGE - variable length
+        The NTLMv2_CLIENT_CHALLENGE structure defines the client challenge in
+        the AUTHENTICATE_MESSAGE. This structure is used only when NTLM v2
+        authentication is configured and is transported in the NTLMv2_RESPONSE
+        structure.
+
+        The method to create this structure is defined in 3.3.2 NTLMv2 Authentication.
+        In this method this variable is known as the temp value. The target_info variable
+        corresponds to the ServerName variable used in that documentation. This is in
+        reality a lot more than just the ServerName and contains the AV_PAIRS structure
+        we need to transport with the message like Channel Binding tokens and others.
+        By default this will be the target_info returned from the CHALLENGE_MESSAGE plus
+        MSV_AV_CHANNEL_BINDINGS if specified otherwise it is a new target_info set with
+        MSV_AV_TIMESTAMP to the current time.
+
+        :param timestamp: An 8-byte timestamp in windows format, 100 nanoseconds since 1601-01-01
+        :param client_challenge: A random 8-byte response generated by the client for the AUTHENTICATE_MESSAGE
+        :param target_info: The target_info structure from the CHALLENGE_MESSAGE with the CBT attached if required
+        :return temp: The CLIENT_CHALLENGE structure that will be added to the NtChallengeResponse structure
+        """
+        resp_type = b'\1'
+        hi_resp_type = b'\1'
+        reserved1 = b'\0' * 2
+        reserved2 = b'\0' * 4
+        reserved3 = b'\0' * 4
+        reserved4 = b'\0' * 4  # This byte is not in the structure defined in 2.2.2.7 but is in the computation guide, works with it present
+
+        temp = resp_type + hi_resp_type + reserved1 + \
+               reserved2 + \
+               timestamp + \
+               client_challenge + \
+               reserved3 + \
+               target_info.get_data() + reserved4
+
+        return temp
+
+    @staticmethod
+    def _calc_resp(password_hash, server_challenge):
+        """
+        Generate the LM response given a 16-byte password hash and the challenge
+        from the CHALLENGE_MESSAGE
+
+        :param password_hash: A 16-byte password hash
+        :param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
+        :return res: A 24-byte buffer to contain the LM response upon return
+        """
+
+        # padding with zeros to make the hash 21 bytes long
+        password_hash += b'\0' * (21 - len(password_hash))
+
+        res = b''
+        dobj = des.DES(password_hash[0:7])
+        res = res + dobj.encrypt(server_challenge[0:8])
+
+        dobj = des.DES(password_hash[7:14])
+        res = res + dobj.encrypt(server_challenge[0:8])
+
+        dobj = des.DES(password_hash[14:21])
+        res = res + dobj.encrypt(server_challenge[0:8])
+        return res
+
+    @staticmethod
+    def _get_channel_bindings_value(server_certificate_hash):
+        """
+        https://msdn.microsoft.com/en-us/library/windows/desktop/dd919963%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
+        https://blogs.msdn.microsoft.com/openspecification/2013/03/26/ntlm-and-channel-binding-hash-aka-extended-protection-for-authentication/
+
+        Get's the MD5 hash of the gss_channel_bindings_struct to add to the AV_PAIR MSV_AV_CHANNEL_BINDINGS.
+        This method takes in the SHA256 hash (Hash of the DER encoded certificate of the server we are connecting to)
+        and add's it to the gss_channel_bindings_struct. It then gets the MD5 hash and converts this to a
+        byte array in preparation of adding it to the AV_PAIR structure.
+
+        :param server_certificate_hash: The SHA256 hash of the server certificate (DER encoded) NTLM is authenticated to
+        :return channel_bindings: An MD5 hash of the gss_channel_bindings_struct to add to the AV_PAIR MsvChannelBindings
+        """
+        # Channel Binding Tokens support, used for NTLMv2
+        # Decode the SHA256 certificate hash
+        certificate_digest = base64.b16decode(server_certificate_hash)
+
+        # Initialise the GssChannelBindingsStruct and add the certificate_digest to the application_data field
+        gss_channel_bindings = GssChannelBindingsStruct()
+        gss_channel_bindings[gss_channel_bindings.APPLICATION_DATA] = 'tls-server-end-point:'.encode() + certificate_digest
+
+        # Get the gss_channel_bindings_struct and create an MD5 hash
+        channel_bindings_struct_data = gss_channel_bindings.get_data()
+        channel_bindings_hash = hashlib.md5(channel_bindings_struct_data).hexdigest()
+
+        try:
+            cbt_value = bytearray.fromhex(channel_bindings_hash)
+        except TypeError:
+            # Work-around for Python 2.6 bug
+            cbt_value = bytearray.fromhex(unicode(channel_bindings_hash))
+
+        channel_bindings = bytes(cbt_value)
+        return channel_bindings
+
+
+def get_windows_timestamp():
+    # Get Windows Date time, 100 nanoseconds since 1601-01-01 in a 64 bit structure
+    timestamp = struct.pack('<q', (116444736000000000 + calendar.timegm(time.gmtime()) * 10000000))
+
+    return timestamp

packages/wakatime/packages/ntlm_auth/constants.py

@@ -0,0 +1,92 @@
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+"""
+    [MS-NLMP] v28.0 2016-07-14
+
+    2.2 Message Syntax
+    The signature field used in NTLM messages
+"""
+NTLM_SIGNATURE = b'NTLMSSP\0'
+
+"""
+    [MS-NLMP] v28.0 2016-07-14
+
+    2.2 Message Syntax
+    The 3 message type options you can have in a message.
+"""
+class MessageTypes(object):
+    NTLM_NEGOTIATE      = 0x1
+    NTLM_CHALLENGE      = 0x2
+    NTLM_AUTHENTICATE   = 0x3
+
+"""
+    [MS-NLMP] v28.0 2016-07-14
+
+    2.2.2.1 AV_PAIR (MsvAvFlags)
+    A 32-bit value indicated server or client configuration
+"""
+class AvFlags(object):
+    AUTHENTICATION_CONSTRAINED  = 0x1
+    MIC_PROVIDED                = 0x2
+    UNTRUSTED_SPN_SOURCE        = 0x4
+
+"""
+    [MS-NLMP] v28.0 2016-07-14
+
+    2.2.2.5 NEGOTIATE
+    During NTLM authentication, each of the following flags is a possible value of the
+    NegotiateFlags field of the NEGOTIATE_MESSAGE, CHALLENGE_MESSAGE and AUTHENTICATE_MESSAGE,
+    unless otherwise noted. These flags define client or server NTLM capabilities
+    supported by the sender.
+"""
+class NegotiateFlags(object):
+    NTLMSSP_NEGOTIATE_56                            = 0x80000000
+    NTLMSSP_NEGOTIATE_KEY_EXCH                      = 0x40000000
+    NTLMSSP_NEGOTIATE_128                           = 0x20000000
+    NTLMSSP_RESERVED_R1                             = 0x10000000
+    NTLMSSP_RESERVED_R2                             = 0x08000000
+    NTLMSSP_RESERVED_R3                             = 0x04000000
+    NTLMSSP_NEGOTIATE_VERSION                       = 0x02000000
+    NTLMSSP_RESERVED_R4                             = 0x01000000
+    NTLMSSP_NEGOTIATE_TARGET_INFO                   = 0x00800000
+    NTLMSSP_REQUEST_NON_NT_SESSION_KEY              = 0x00400000
+    NTLMSSP_RESERVED_R5                             = 0x00200000
+    NTLMSSP_NEGOTIATE_IDENTITY                      = 0x00100000
+    NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY      = 0x00080000
+    NTLMSSP_RESERVED_R6                             = 0x00040000
+    NTLMSSP_TARGET_TYPE_SERVER                      = 0x00020000
+    NTLMSSP_TARGET_TYPE_DOMAIN                      = 0x00010000
+    NTLMSSP_NEGOTIATE_ALWAYS_SIGN                   = 0x00008000
+    NTLMSSP_RESERVED_R7                             = 0x00004000
+    NTLMSSP_NEGOTIATE_OEM_WORKSTATION_SUPPLIED      = 0x00002000
+    NTLMSSP_NEGOTIATE_OEM_DOMAIN_SUPPLIED           = 0x00001000
+    NTLMSSP_ANOYNMOUS                               = 0x00000800
+    NTLMSSP_RESERVED_R8                             = 0x00000400
+    NTLMSSP_NEGOTIATE_NTLM                          = 0x00000200
+    NTLMSSP_RESERVED_R9                             = 0x00000100
+    NTLMSSP_NEGOTIATE_LM_KEY                        = 0x00000080
+    NTLMSSP_NEGOTIATE_DATAGRAM                      = 0x00000040
+    NTLMSSP_NEGOTIATE_SEAL                          = 0x00000020
+    NTLMSSP_NEGOTIATE_SIGN                          = 0x00000010
+    NTLMSSP_RESERVED_R10                            = 0x00000008
+    NTLMSSP_REQUEST_TARGET                          = 0x00000004
+    NTLMSSP_NEGOTIATE_OEM                           = 0x00000002
+    NTLMSSP_NEGOTIATE_UNICODE                       = 0x00000001
+
+class SignSealConstants(object):
+    # Magic Contants used to get the signing and sealing key for Extended Session Security
+    CLIENT_SIGNING = b"session key to client-to-server signing key magic constant\0"
+    SERVER_SIGNING = b"session key to server-to-client signing key magic constant\0"
+    CLIENT_SEALING = b"session key to client-to-server sealing key magic constant\0"
+    SERVER_SEALING = b"session key to server-to-client sealing key magic constant\0"

packages/wakatime/packages/ntlm_auth/des.py

@@ -0,0 +1,88 @@
+# This file is part of 'NTLM Authorization Proxy Server' http://sourceforge.net/projects/ntlmaps/
+# Copyright 2001 Dmitry A. Rozmanov <dima@xenon.spb.ru>
+#
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+import logging
+import six
+from ntlm_auth import des_c
+
+log = logging.getLogger(__name__)
+
+
+class DES:
+    des_c_obj = None
+
+    def __init__(self, key_str):
+        k = str_to_key56(key_str)
+        k = key56_to_key64(k)
+
+        key_str = b''
+        for i in k:
+            key_str += six.int2byte(i & 0xFF)
+
+        self.des_c_obj = des_c.DES(key_str)
+
+    def encrypt(self, plain_text):
+        return self.des_c_obj.encrypt(plain_text)
+
+    def decrypt(self, crypted_text):
+        return self.des_c_obj.decrypt(crypted_text)
+
+
+DESException = 'DESException'
+
+
+def str_to_key56(key_str):
+
+    if not type(key_str) == six.binary_type:
+        # TODO rsanders high - figure out how to make this not necessary
+        key_str = key_str.encode('ascii')
+
+    if len(key_str) < 7:
+        key_str = key_str + b'\000\000\000\000\000\000\000'[:(7 - len(key_str))]
+    key_56 = []
+    for i in six.iterbytes(key_str[:7]):
+        key_56.append(i)
+
+    return key_56
+
+
+def key56_to_key64(key_56):
+    key = []
+    for i in range(8):
+        key.append(0)
+
+    key[0] = key_56[0]
+    key[1] = ((key_56[0] << 7) & 0xFF) | (key_56[1] >> 1)
+    key[2] = ((key_56[1] << 6) & 0xFF) | (key_56[2] >> 2)
+    key[3] = ((key_56[2] << 5) & 0xFF) | (key_56[3] >> 3)
+    key[4] = ((key_56[3] << 4) & 0xFF) | (key_56[4] >> 4)
+    key[5] = ((key_56[4] << 3) & 0xFF) | (key_56[5] >> 5)
+    key[6] = ((key_56[5] << 2) & 0xFF) | (key_56[6] >> 6)
+    key[7] = (key_56[6] << 1) & 0xFF
+
+    key = set_key_odd_parity(key)
+
+    return key
+
+
+def set_key_odd_parity(key):
+    for i in range(len(key)):
+        for k in range(7):
+            bit = 0
+            t = key[i] >> k
+            bit = (t ^ bit) & 0x1
+        key[i] = (key[i] & 0xFE) | bit
+
+    return key

packages/wakatime/packages/ntlm_auth/des_c.py

@@ -0,0 +1,254 @@
+# This file is part of 'NTLM Authorization Proxy Server' http://sourceforge.net/projects/ntlmaps/
+# Copyright 2001 Dmitry A. Rozmanov <dima@xenon.spb.ru>
+#
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+import six
+
+from ntlm_auth.U32 import U32
+from ntlm_auth.des_data import des_SPtrans, des_skb
+
+def c2l(c):
+    "char[4] to unsigned long"
+    l = U32(c[0])
+    l = l | (U32(c[1]) << 8)
+    l = l | (U32(c[2]) << 16)
+    l = l | (U32(c[3]) << 24)
+    return l
+
+
+def l2c(l):
+    "unsigned long to char[4]"
+    c = []
+    c.append(int(l & U32(0xFF)))
+    c.append(int((l >> 8) & U32(0xFF)))
+    c.append(int((l >> 16) & U32(0xFF)))
+    c.append(int((l >> 24) & U32(0xFF)))
+    return c
+
+
+def D_ENCRYPT(tup, u, t, s):
+    L, R, S = tup
+    # print 'LRS1', L, R, S, u, t, '-->',
+    u = (R ^ s[S])
+    t = R ^ s[S + 1]
+    t = ((t >> 4) + (t << 28))
+    L = L ^ (des_SPtrans[1][int((t) & U32(0x3f))] |
+             des_SPtrans[3][int((t >> 8) & U32(0x3f))] |
+             des_SPtrans[5][int((t >> 16) & U32(0x3f))] |
+             des_SPtrans[7][int((t >> 24) & U32(0x3f))] |
+             des_SPtrans[0][int((u) & U32(0x3f))] |
+             des_SPtrans[2][int((u >> 8) & U32(0x3f))] |
+             des_SPtrans[4][int((u >> 16) & U32(0x3f))] |
+             des_SPtrans[6][int((u >> 24) & U32(0x3f))])
+    # print 'LRS:', L, R, S, u, t
+    return (L, R, S), u, t, s
+
+
+def PERM_OP(tup, n, m):
+    "tup - (a, b, t)"
+    a, b, t = tup
+    t = ((a >> n) ^ b) & m
+    b = b ^ t
+    a = a ^ (t << n)
+    return (a, b, t)
+
+
+def HPERM_OP(tup, n, m):
+    "tup - (a, t)"
+    a, t = tup
+    t = ((a << (16 - n)) ^ a) & m
+    a = a ^ t ^ (t >> (16 - n))
+    return a, t
+
+
+shifts2 = [0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0]
+
+
+class DES:
+    KeySched = None  # des_key_schedule
+
+    def __init__(self, key_str):
+        self.KeySched = des_set_key(key_str)
+
+    def decrypt(self, str):
+        # block - UChar[]
+        block = []
+
+        for i in six.iterbytes(str):
+            block.append(i)
+
+        # print block
+        block = des_ecb_encrypt(block, self.KeySched, 0)
+
+        res = b''
+        for i in block:
+            res = res + six.int2byte(i)
+
+        return res
+
+    def encrypt(self, plaintext):
+        # block - UChar[]
+
+        block = []
+        for i in plaintext:
+            block.append(i)
+
+        block = des_ecb_encrypt(block, self.KeySched, 1)
+
+        res = b''
+
+        for i in block:
+            res += six.int2byte(i)
+
+        return res
+
+
+def des_encript(input, ks, encrypt):
+    # input - U32[]
+    # output - U32[]
+    # ks - des_key_shedule - U32[2][16]
+    # encrypt - int
+    # l, r, t, u - U32
+    # i - int
+    # s - U32[]
+
+    l = input[0]
+    r = input[1]
+    t = U32(0)
+    u = U32(0)
+
+    r, l, t = PERM_OP((r, l, t), 4, U32(0x0f0f0f0f))
+    l, r, t = PERM_OP((l, r, t), 16, U32(0x0000ffff))
+    r, l, t = PERM_OP((r, l, t), 2, U32(0x33333333))
+    l, r, t = PERM_OP((l, r, t), 8, U32(0x00ff00ff))
+    r, l, t = PERM_OP((r, l, t), 1, U32(0x55555555))
+
+    t = (r << 1) | (r >> 31)
+    r = (l << 1) | (l >> 31)
+    l = t
+
+    s = ks  # ???????????????
+    # print l, r
+    if encrypt:
+        for i in range(0, 32, 4):
+            rtup, u, t, s = D_ENCRYPT((l, r, i + 0), u, t, s)
+            l = rtup[0]
+            r = rtup[1]
+            rtup, u, t, s = D_ENCRYPT((r, l, i + 2), u, t, s)
+            r = rtup[0]
+            l = rtup[1]
+    else:
+        for i in range(30, 0, -4):
+            rtup, u, t, s = D_ENCRYPT((l, r, i - 0), u, t, s)
+            l = rtup[0]
+            r = rtup[1]
+            rtup, u, t, s = D_ENCRYPT((r, l, i - 2), u, t, s)
+            r = rtup[0]
+            l = rtup[1]
+        # print l, r
+    l = (l >> 1) | (l << 31)
+    r = (r >> 1) | (r << 31)
+
+    r, l, t = PERM_OP((r, l, t), 1, U32(0x55555555))
+    l, r, t = PERM_OP((l, r, t), 8, U32(0x00ff00ff))
+    r, l, t = PERM_OP((r, l, t), 2, U32(0x33333333))
+    l, r, t = PERM_OP((l, r, t), 16, U32(0x0000ffff))
+    r, l, t = PERM_OP((r, l, t), 4, U32(0x0f0f0f0f))
+
+    output = [l]
+    output.append(r)
+    l, r, t, u = U32(0), U32(0), U32(0), U32(0)
+    return output
+
+
+def des_ecb_encrypt(input, ks, encrypt):
+    # input - des_cblock - UChar[8]
+    # output - des_cblock - UChar[8]
+    # ks - des_key_shedule - U32[2][16]
+    # encrypt - int
+
+    # print input
+    l0 = c2l(input[0:4])
+    l1 = c2l(input[4:8])
+    ll = [l0]
+    ll.append(l1)
+    # print ll
+    ll = des_encript(ll, ks, encrypt)
+    # print ll
+    l0 = ll[0]
+    l1 = ll[1]
+    output = l2c(l0)
+    output = output + l2c(l1)
+    # print output
+    l0, l1, ll[0], ll[1] = U32(0), U32(0), U32(0), U32(0)
+    return output
+
+
+def des_set_key(key):
+    # key - des_cblock - UChar[8]
+    # schedule - des_key_schedule
+
+    # register unsigned long c,d,t,s;
+    # register unsigned char *in;
+    # register unsigned long *k;
+    # register int i;
+
+    # k = schedule
+    # in = key
+
+    k = []
+    c = c2l(key[0:4])
+    d = c2l(key[4:8])
+    t = U32(0)
+
+    d, c, t = PERM_OP((d, c, t), 4, U32(0x0f0f0f0f))
+    c, t = HPERM_OP((c, t), -2, U32(0xcccc0000))
+    d, t = HPERM_OP((d, t), -2, U32(0xcccc0000))
+    d, c, t = PERM_OP((d, c, t), 1, U32(0x55555555))
+    c, d, t = PERM_OP((c, d, t), 8, U32(0x00ff00ff))
+    d, c, t = PERM_OP((d, c, t), 1, U32(0x55555555))
+
+    d = (((d & U32(0x000000ff)) << 16) | (d & U32(0x0000ff00)) | ((d & U32(0x00ff0000)) >> 16) | (
+        (c & U32(0xf0000000)) >> 4))
+    c = c & U32(0x0fffffff)
+
+    for i in range(16):
+        if (shifts2[i]):
+            c = ((c >> 2) | (c << 26))
+            d = ((d >> 2) | (d << 26))
+        else:
+            c = ((c >> 1) | (c << 27))
+            d = ((d >> 1) | (d << 27))
+        c = c & U32(0x0fffffff)
+        d = d & U32(0x0fffffff)
+
+        s = des_skb[0][int((c) & U32(0x3f))] | \
+            des_skb[1][int(((c >> 6) & U32(0x03)) | ((c >> 7) & U32(0x3c)))] | \
+            des_skb[2][int(((c >> 13) & U32(0x0f)) | ((c >> 14) & U32(0x30)))] | \
+            des_skb[3][int(((c >> 20) & U32(0x01)) | ((c >> 21) & U32(0x06)) | ((c >> 22) & U32(0x38)))]
+
+        t = des_skb[4][int((d) & U32(0x3f))] | \
+            des_skb[5][int(((d >> 7) & U32(0x03)) | ((d >> 8) & U32(0x3c)))] | \
+            des_skb[6][int((d >> 15) & U32(0x3f))] | \
+            des_skb[7][int(((d >> 21) & U32(0x0f)) | ((d >> 22) & U32(0x30)))]
+        # print s, t
+
+        k.append(((t << 16) | (s & U32(0x0000ffff))) & U32(0xffffffff))
+        s = ((s >> 16) | (t & U32(0xffff0000)))
+        s = (s << 4) | (s >> 28)
+        k.append(s & U32(0xffffffff))
+
+    schedule = k
+
+    return schedule

packages/wakatime/packages/ntlm_auth/des_data.py

@@ -0,0 +1,348 @@
+# This file is part of 'NTLM Authorization Proxy Server' http://sourceforge.net/projects/ntlmaps/
+# Copyright 2001 Dmitry A. Rozmanov <dima@xenon.spb.ru>
+#
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+from ntlm_auth.U32 import U32
+
+# static unsigned long des_SPtrans[8][64]={
+
+des_SPtrans = \
+    [
+        # nibble 0
+        [
+            U32(0x00820200), U32(0x00020000), U32(0x80800000), U32(0x80820200),
+            U32(0x00800000), U32(0x80020200), U32(0x80020000), U32(0x80800000),
+            U32(0x80020200), U32(0x00820200), U32(0x00820000), U32(0x80000200),
+            U32(0x80800200), U32(0x00800000), U32(0x00000000), U32(0x80020000),
+            U32(0x00020000), U32(0x80000000), U32(0x00800200), U32(0x00020200),
+            U32(0x80820200), U32(0x00820000), U32(0x80000200), U32(0x00800200),
+            U32(0x80000000), U32(0x00000200), U32(0x00020200), U32(0x80820000),
+            U32(0x00000200), U32(0x80800200), U32(0x80820000), U32(0x00000000),
+            U32(0x00000000), U32(0x80820200), U32(0x00800200), U32(0x80020000),
+            U32(0x00820200), U32(0x00020000), U32(0x80000200), U32(0x00800200),
+            U32(0x80820000), U32(0x00000200), U32(0x00020200), U32(0x80800000),
+            U32(0x80020200), U32(0x80000000), U32(0x80800000), U32(0x00820000),
+            U32(0x80820200), U32(0x00020200), U32(0x00820000), U32(0x80800200),
+            U32(0x00800000), U32(0x80000200), U32(0x80020000), U32(0x00000000),
+            U32(0x00020000), U32(0x00800000), U32(0x80800200), U32(0x00820200),
+            U32(0x80000000), U32(0x80820000), U32(0x00000200), U32(0x80020200),
+        ],
+
+        # nibble 1
+        [
+            U32(0x10042004), U32(0x00000000), U32(0x00042000), U32(0x10040000),
+            U32(0x10000004), U32(0x00002004), U32(0x10002000), U32(0x00042000),
+            U32(0x00002000), U32(0x10040004), U32(0x00000004), U32(0x10002000),
+            U32(0x00040004), U32(0x10042000), U32(0x10040000), U32(0x00000004),
+            U32(0x00040000), U32(0x10002004), U32(0x10040004), U32(0x00002000),
+            U32(0x00042004), U32(0x10000000), U32(0x00000000), U32(0x00040004),
+            U32(0x10002004), U32(0x00042004), U32(0x10042000), U32(0x10000004),
+            U32(0x10000000), U32(0x00040000), U32(0x00002004), U32(0x10042004),
+            U32(0x00040004), U32(0x10042000), U32(0x10002000), U32(0x00042004),
+            U32(0x10042004), U32(0x00040004), U32(0x10000004), U32(0x00000000),
+            U32(0x10000000), U32(0x00002004), U32(0x00040000), U32(0x10040004),
+            U32(0x00002000), U32(0x10000000), U32(0x00042004), U32(0x10002004),
+            U32(0x10042000), U32(0x00002000), U32(0x00000000), U32(0x10000004),
+            U32(0x00000004), U32(0x10042004), U32(0x00042000), U32(0x10040000),
+            U32(0x10040004), U32(0x00040000), U32(0x00002004), U32(0x10002000),
+            U32(0x10002004), U32(0x00000004), U32(0x10040000), U32(0x00042000),
+        ],
+
+        # nibble 2
+        [
+            U32(0x41000000), U32(0x01010040), U32(0x00000040), U32(0x41000040),
+            U32(0x40010000), U32(0x01000000), U32(0x41000040), U32(0x00010040),
+            U32(0x01000040), U32(0x00010000), U32(0x01010000), U32(0x40000000),
+            U32(0x41010040), U32(0x40000040), U32(0x40000000), U32(0x41010000),
+            U32(0x00000000), U32(0x40010000), U32(0x01010040), U32(0x00000040),
+            U32(0x40000040), U32(0x41010040), U32(0x00010000), U32(0x41000000),
+            U32(0x41010000), U32(0x01000040), U32(0x40010040), U32(0x01010000),
+            U32(0x00010040), U32(0x00000000), U32(0x01000000), U32(0x40010040),
+            U32(0x01010040), U32(0x00000040), U32(0x40000000), U32(0x00010000),
+            U32(0x40000040), U32(0x40010000), U32(0x01010000), U32(0x41000040),
+            U32(0x00000000), U32(0x01010040), U32(0x00010040), U32(0x41010000),
+            U32(0x40010000), U32(0x01000000), U32(0x41010040), U32(0x40000000),
+            U32(0x40010040), U32(0x41000000), U32(0x01000000), U32(0x41010040),
+            U32(0x00010000), U32(0x01000040), U32(0x41000040), U32(0x00010040),
+            U32(0x01000040), U32(0x00000000), U32(0x41010000), U32(0x40000040),
+            U32(0x41000000), U32(0x40010040), U32(0x00000040), U32(0x01010000),
+        ],
+
+        # nibble 3
+        [
+            U32(0x00100402), U32(0x04000400), U32(0x00000002), U32(0x04100402),
+            U32(0x00000000), U32(0x04100000), U32(0x04000402), U32(0x00100002),
+            U32(0x04100400), U32(0x04000002), U32(0x04000000), U32(0x00000402),
+            U32(0x04000002), U32(0x00100402), U32(0x00100000), U32(0x04000000),
+            U32(0x04100002), U32(0x00100400), U32(0x00000400), U32(0x00000002),
+            U32(0x00100400), U32(0x04000402), U32(0x04100000), U32(0x00000400),
+            U32(0x00000402), U32(0x00000000), U32(0x00100002), U32(0x04100400),
+            U32(0x04000400), U32(0x04100002), U32(0x04100402), U32(0x00100000),
+            U32(0x04100002), U32(0x00000402), U32(0x00100000), U32(0x04000002),
+            U32(0x00100400), U32(0x04000400), U32(0x00000002), U32(0x04100000),
+            U32(0x04000402), U32(0x00000000), U32(0x00000400), U32(0x00100002),
+            U32(0x00000000), U32(0x04100002), U32(0x04100400), U32(0x00000400),
+            U32(0x04000000), U32(0x04100402), U32(0x00100402), U32(0x00100000),
+            U32(0x04100402), U32(0x00000002), U32(0x04000400), U32(0x00100402),
+            U32(0x00100002), U32(0x00100400), U32(0x04100000), U32(0x04000402),
+            U32(0x00000402), U32(0x04000000), U32(0x04000002), U32(0x04100400),
+        ],
+
+        # nibble 4
+        [
+            U32(0x02000000), U32(0x00004000), U32(0x00000100), U32(0x02004108),
+            U32(0x02004008), U32(0x02000100), U32(0x00004108), U32(0x02004000),
+            U32(0x00004000), U32(0x00000008), U32(0x02000008), U32(0x00004100),
+            U32(0x02000108), U32(0x02004008), U32(0x02004100), U32(0x00000000),
+            U32(0x00004100), U32(0x02000000), U32(0x00004008), U32(0x00000108),
+            U32(0x02000100), U32(0x00004108), U32(0x00000000), U32(0x02000008),
+            U32(0x00000008), U32(0x02000108), U32(0x02004108), U32(0x00004008),
+            U32(0x02004000), U32(0x00000100), U32(0x00000108), U32(0x02004100),
+            U32(0x02004100), U32(0x02000108), U32(0x00004008), U32(0x02004000),
+            U32(0x00004000), U32(0x00000008), U32(0x02000008), U32(0x02000100),
+            U32(0x02000000), U32(0x00004100), U32(0x02004108), U32(0x00000000),
+            U32(0x00004108), U32(0x02000000), U32(0x00000100), U32(0x00004008),
+            U32(0x02000108), U32(0x00000100), U32(0x00000000), U32(0x02004108),
+            U32(0x02004008), U32(0x02004100), U32(0x00000108), U32(0x00004000),
+            U32(0x00004100), U32(0x02004008), U32(0x02000100), U32(0x00000108),
+            U32(0x00000008), U32(0x00004108), U32(0x02004000), U32(0x02000008),
+        ],
+
+        # nibble 5
+        [
+            U32(0x20000010), U32(0x00080010), U32(0x00000000), U32(0x20080800),
+            U32(0x00080010), U32(0x00000800), U32(0x20000810), U32(0x00080000),
+            U32(0x00000810), U32(0x20080810), U32(0x00080800), U32(0x20000000),
+            U32(0x20000800), U32(0x20000010), U32(0x20080000), U32(0x00080810),
+            U32(0x00080000), U32(0x20000810), U32(0x20080010), U32(0x00000000),
+            U32(0x00000800), U32(0x00000010), U32(0x20080800), U32(0x20080010),
+            U32(0x20080810), U32(0x20080000), U32(0x20000000), U32(0x00000810),
+            U32(0x00000010), U32(0x00080800), U32(0x00080810), U32(0x20000800),
+            U32(0x00000810), U32(0x20000000), U32(0x20000800), U32(0x00080810),
+            U32(0x20080800), U32(0x00080010), U32(0x00000000), U32(0x20000800),
+            U32(0x20000000), U32(0x00000800), U32(0x20080010), U32(0x00080000),
+            U32(0x00080010), U32(0x20080810), U32(0x00080800), U32(0x00000010),
+            U32(0x20080810), U32(0x00080800), U32(0x00080000), U32(0x20000810),
+            U32(0x20000010), U32(0x20080000), U32(0x00080810), U32(0x00000000),
+            U32(0x00000800), U32(0x20000010), U32(0x20000810), U32(0x20080800),
+            U32(0x20080000), U32(0x00000810), U32(0x00000010), U32(0x20080010),
+        ],
+
+        # nibble 6
+        [
+            U32(0x00001000), U32(0x00000080), U32(0x00400080), U32(0x00400001),
+            U32(0x00401081), U32(0x00001001), U32(0x00001080), U32(0x00000000),
+            U32(0x00400000), U32(0x00400081), U32(0x00000081), U32(0x00401000),
+            U32(0x00000001), U32(0x00401080), U32(0x00401000), U32(0x00000081),
+            U32(0x00400081), U32(0x00001000), U32(0x00001001), U32(0x00401081),
+            U32(0x00000000), U32(0x00400080), U32(0x00400001), U32(0x00001080),
+            U32(0x00401001), U32(0x00001081), U32(0x00401080), U32(0x00000001),
+            U32(0x00001081), U32(0x00401001), U32(0x00000080), U32(0x00400000),
+            U32(0x00001081), U32(0x00401000), U32(0x00401001), U32(0x00000081),
+            U32(0x00001000), U32(0x00000080), U32(0x00400000), U32(0x00401001),
+            U32(0x00400081), U32(0x00001081), U32(0x00001080), U32(0x00000000),
+            U32(0x00000080), U32(0x00400001), U32(0x00000001), U32(0x00400080),
+            U32(0x00000000), U32(0x00400081), U32(0x00400080), U32(0x00001080),
+            U32(0x00000081), U32(0x00001000), U32(0x00401081), U32(0x00400000),
+            U32(0x00401080), U32(0x00000001), U32(0x00001001), U32(0x00401081),
+            U32(0x00400001), U32(0x00401080), U32(0x00401000), U32(0x00001001),
+        ],
+
+        # nibble 7
+        [
+            U32(0x08200020), U32(0x08208000), U32(0x00008020), U32(0x00000000),
+            U32(0x08008000), U32(0x00200020), U32(0x08200000), U32(0x08208020),
+            U32(0x00000020), U32(0x08000000), U32(0x00208000), U32(0x00008020),
+            U32(0x00208020), U32(0x08008020), U32(0x08000020), U32(0x08200000),
+            U32(0x00008000), U32(0x00208020), U32(0x00200020), U32(0x08008000),
+            U32(0x08208020), U32(0x08000020), U32(0x00000000), U32(0x00208000),
+            U32(0x08000000), U32(0x00200000), U32(0x08008020), U32(0x08200020),
+            U32(0x00200000), U32(0x00008000), U32(0x08208000), U32(0x00000020),
+            U32(0x00200000), U32(0x00008000), U32(0x08000020), U32(0x08208020),
+            U32(0x00008020), U32(0x08000000), U32(0x00000000), U32(0x00208000),
+            U32(0x08200020), U32(0x08008020), U32(0x08008000), U32(0x00200020),
+            U32(0x08208000), U32(0x00000020), U32(0x00200020), U32(0x08008000),
+            U32(0x08208020), U32(0x00200000), U32(0x08200000), U32(0x08000020),
+            U32(0x00208000), U32(0x00008020), U32(0x08008020), U32(0x08200000),
+            U32(0x00000020), U32(0x08208000), U32(0x00208020), U32(0x00000000),
+            U32(0x08000000), U32(0x08200020), U32(0x00008000), U32(0x00208020),
+        ],
+    ]
+
+# static unsigned long des_skb[8][64]={
+
+des_skb = \
+    [
+        # for C bits (numbered as per FIPS 46) 1 2 3 4 5 6
+        [
+            U32(0x00000000), U32(0x00000010), U32(0x20000000), U32(0x20000010),
+            U32(0x00010000), U32(0x00010010), U32(0x20010000), U32(0x20010010),
+            U32(0x00000800), U32(0x00000810), U32(0x20000800), U32(0x20000810),
+            U32(0x00010800), U32(0x00010810), U32(0x20010800), U32(0x20010810),
+            U32(0x00000020), U32(0x00000030), U32(0x20000020), U32(0x20000030),
+            U32(0x00010020), U32(0x00010030), U32(0x20010020), U32(0x20010030),
+            U32(0x00000820), U32(0x00000830), U32(0x20000820), U32(0x20000830),
+            U32(0x00010820), U32(0x00010830), U32(0x20010820), U32(0x20010830),
+            U32(0x00080000), U32(0x00080010), U32(0x20080000), U32(0x20080010),
+            U32(0x00090000), U32(0x00090010), U32(0x20090000), U32(0x20090010),
+            U32(0x00080800), U32(0x00080810), U32(0x20080800), U32(0x20080810),
+            U32(0x00090800), U32(0x00090810), U32(0x20090800), U32(0x20090810),
+            U32(0x00080020), U32(0x00080030), U32(0x20080020), U32(0x20080030),
+            U32(0x00090020), U32(0x00090030), U32(0x20090020), U32(0x20090030),
+            U32(0x00080820), U32(0x00080830), U32(0x20080820), U32(0x20080830),
+            U32(0x00090820), U32(0x00090830), U32(0x20090820), U32(0x20090830),
+        ],
+
+        # for C bits (numbered as per FIPS 46) 7 8 10 11 12 13
+        [
+            U32(0x00000000), U32(0x02000000), U32(0x00002000), U32(0x02002000),
+            U32(0x00200000), U32(0x02200000), U32(0x00202000), U32(0x02202000),
+            U32(0x00000004), U32(0x02000004), U32(0x00002004), U32(0x02002004),
+            U32(0x00200004), U32(0x02200004), U32(0x00202004), U32(0x02202004),
+            U32(0x00000400), U32(0x02000400), U32(0x00002400), U32(0x02002400),
+            U32(0x00200400), U32(0x02200400), U32(0x00202400), U32(0x02202400),
+            U32(0x00000404), U32(0x02000404), U32(0x00002404), U32(0x02002404),
+            U32(0x00200404), U32(0x02200404), U32(0x00202404), U32(0x02202404),
+            U32(0x10000000), U32(0x12000000), U32(0x10002000), U32(0x12002000),
+            U32(0x10200000), U32(0x12200000), U32(0x10202000), U32(0x12202000),
+            U32(0x10000004), U32(0x12000004), U32(0x10002004), U32(0x12002004),
+            U32(0x10200004), U32(0x12200004), U32(0x10202004), U32(0x12202004),
+            U32(0x10000400), U32(0x12000400), U32(0x10002400), U32(0x12002400),
+            U32(0x10200400), U32(0x12200400), U32(0x10202400), U32(0x12202400),
+            U32(0x10000404), U32(0x12000404), U32(0x10002404), U32(0x12002404),
+            U32(0x10200404), U32(0x12200404), U32(0x10202404), U32(0x12202404),
+        ],
+
+        # for C bits (numbered as per FIPS 46) 14 15 16 17 19 20
+        [
+            U32(0x00000000), U32(0x00000001), U32(0x00040000), U32(0x00040001),
+            U32(0x01000000), U32(0x01000001), U32(0x01040000), U32(0x01040001),
+            U32(0x00000002), U32(0x00000003), U32(0x00040002), U32(0x00040003),
+            U32(0x01000002), U32(0x01000003), U32(0x01040002), U32(0x01040003),
+            U32(0x00000200), U32(0x00000201), U32(0x00040200), U32(0x00040201),
+            U32(0x01000200), U32(0x01000201), U32(0x01040200), U32(0x01040201),
+            U32(0x00000202), U32(0x00000203), U32(0x00040202), U32(0x00040203),
+            U32(0x01000202), U32(0x01000203), U32(0x01040202), U32(0x01040203),
+            U32(0x08000000), U32(0x08000001), U32(0x08040000), U32(0x08040001),
+            U32(0x09000000), U32(0x09000001), U32(0x09040000), U32(0x09040001),
+            U32(0x08000002), U32(0x08000003), U32(0x08040002), U32(0x08040003),
+            U32(0x09000002), U32(0x09000003), U32(0x09040002), U32(0x09040003),
+            U32(0x08000200), U32(0x08000201), U32(0x08040200), U32(0x08040201),
+            U32(0x09000200), U32(0x09000201), U32(0x09040200), U32(0x09040201),
+            U32(0x08000202), U32(0x08000203), U32(0x08040202), U32(0x08040203),
+            U32(0x09000202), U32(0x09000203), U32(0x09040202), U32(0x09040203),
+        ],
+
+        # for C bits (numbered as per FIPS 46) 21 23 24 26 27 28
+        [
+            U32(0x00000000), U32(0x00100000), U32(0x00000100), U32(0x00100100),
+            U32(0x00000008), U32(0x00100008), U32(0x00000108), U32(0x00100108),
+            U32(0x00001000), U32(0x00101000), U32(0x00001100), U32(0x00101100),
+            U32(0x00001008), U32(0x00101008), U32(0x00001108), U32(0x00101108),
+            U32(0x04000000), U32(0x04100000), U32(0x04000100), U32(0x04100100),
+            U32(0x04000008), U32(0x04100008), U32(0x04000108), U32(0x04100108),
+            U32(0x04001000), U32(0x04101000), U32(0x04001100), U32(0x04101100),
+            U32(0x04001008), U32(0x04101008), U32(0x04001108), U32(0x04101108),
+            U32(0x00020000), U32(0x00120000), U32(0x00020100), U32(0x00120100),
+            U32(0x00020008), U32(0x00120008), U32(0x00020108), U32(0x00120108),
+            U32(0x00021000), U32(0x00121000), U32(0x00021100), U32(0x00121100),
+            U32(0x00021008), U32(0x00121008), U32(0x00021108), U32(0x00121108),
+            U32(0x04020000), U32(0x04120000), U32(0x04020100), U32(0x04120100),
+            U32(0x04020008), U32(0x04120008), U32(0x04020108), U32(0x04120108),
+            U32(0x04021000), U32(0x04121000), U32(0x04021100), U32(0x04121100),
+            U32(0x04021008), U32(0x04121008), U32(0x04021108), U32(0x04121108),
+        ],
+
+        # for D bits (numbered as per FIPS 46) 1 2 3 4 5 6
+        [
+            U32(0x00000000), U32(0x10000000), U32(0x00010000), U32(0x10010000),
+            U32(0x00000004), U32(0x10000004), U32(0x00010004), U32(0x10010004),
+            U32(0x20000000), U32(0x30000000), U32(0x20010000), U32(0x30010000),
+            U32(0x20000004), U32(0x30000004), U32(0x20010004), U32(0x30010004),
+            U32(0x00100000), U32(0x10100000), U32(0x00110000), U32(0x10110000),
+            U32(0x00100004), U32(0x10100004), U32(0x00110004), U32(0x10110004),
+            U32(0x20100000), U32(0x30100000), U32(0x20110000), U32(0x30110000),
+            U32(0x20100004), U32(0x30100004), U32(0x20110004), U32(0x30110004),
+            U32(0x00001000), U32(0x10001000), U32(0x00011000), U32(0x10011000),
+            U32(0x00001004), U32(0x10001004), U32(0x00011004), U32(0x10011004),
+            U32(0x20001000), U32(0x30001000), U32(0x20011000), U32(0x30011000),
+            U32(0x20001004), U32(0x30001004), U32(0x20011004), U32(0x30011004),
+            U32(0x00101000), U32(0x10101000), U32(0x00111000), U32(0x10111000),
+            U32(0x00101004), U32(0x10101004), U32(0x00111004), U32(0x10111004),
+            U32(0x20101000), U32(0x30101000), U32(0x20111000), U32(0x30111000),
+            U32(0x20101004), U32(0x30101004), U32(0x20111004), U32(0x30111004),
+        ],
+
+        # for D bits (numbered as per FIPS 46) 8 9 11 12 13 14
+        [
+            U32(0x00000000), U32(0x08000000), U32(0x00000008), U32(0x08000008),
+            U32(0x00000400), U32(0x08000400), U32(0x00000408), U32(0x08000408),
+            U32(0x00020000), U32(0x08020000), U32(0x00020008), U32(0x08020008),
+            U32(0x00020400), U32(0x08020400), U32(0x00020408), U32(0x08020408),
+            U32(0x00000001), U32(0x08000001), U32(0x00000009), U32(0x08000009),
+            U32(0x00000401), U32(0x08000401), U32(0x00000409), U32(0x08000409),
+            U32(0x00020001), U32(0x08020001), U32(0x00020009), U32(0x08020009),
+            U32(0x00020401), U32(0x08020401), U32(0x00020409), U32(0x08020409),
+            U32(0x02000000), U32(0x0A000000), U32(0x02000008), U32(0x0A000008),
+            U32(0x02000400), U32(0x0A000400), U32(0x02000408), U32(0x0A000408),
+            U32(0x02020000), U32(0x0A020000), U32(0x02020008), U32(0x0A020008),
+            U32(0x02020400), U32(0x0A020400), U32(0x02020408), U32(0x0A020408),
+            U32(0x02000001), U32(0x0A000001), U32(0x02000009), U32(0x0A000009),
+            U32(0x02000401), U32(0x0A000401), U32(0x02000409), U32(0x0A000409),
+            U32(0x02020001), U32(0x0A020001), U32(0x02020009), U32(0x0A020009),
+            U32(0x02020401), U32(0x0A020401), U32(0x02020409), U32(0x0A020409),
+        ],
+
+        # for D bits (numbered as per FIPS 46) 16 17 18 19 20 21
+        [
+            U32(0x00000000), U32(0x00000100), U32(0x00080000), U32(0x00080100),
+            U32(0x01000000), U32(0x01000100), U32(0x01080000), U32(0x01080100),
+            U32(0x00000010), U32(0x00000110), U32(0x00080010), U32(0x00080110),
+            U32(0x01000010), U32(0x01000110), U32(0x01080010), U32(0x01080110),
+            U32(0x00200000), U32(0x00200100), U32(0x00280000), U32(0x00280100),
+            U32(0x01200000), U32(0x01200100), U32(0x01280000), U32(0x01280100),
+            U32(0x00200010), U32(0x00200110), U32(0x00280010), U32(0x00280110),
+            U32(0x01200010), U32(0x01200110), U32(0x01280010), U32(0x01280110),
+            U32(0x00000200), U32(0x00000300), U32(0x00080200), U32(0x00080300),
+            U32(0x01000200), U32(0x01000300), U32(0x01080200), U32(0x01080300),
+            U32(0x00000210), U32(0x00000310), U32(0x00080210), U32(0x00080310),
+            U32(0x01000210), U32(0x01000310), U32(0x01080210), U32(0x01080310),
+            U32(0x00200200), U32(0x00200300), U32(0x00280200), U32(0x00280300),
+            U32(0x01200200), U32(0x01200300), U32(0x01280200), U32(0x01280300),
+            U32(0x00200210), U32(0x00200310), U32(0x00280210), U32(0x00280310),
+            U32(0x01200210), U32(0x01200310), U32(0x01280210), U32(0x01280310),
+        ],
+
+        # for D bits (numbered as per FIPS 46) 22 23 24 25 27 28
+        [
+            U32(0x00000000), U32(0x04000000), U32(0x00040000), U32(0x04040000),
+            U32(0x00000002), U32(0x04000002), U32(0x00040002), U32(0x04040002),
+            U32(0x00002000), U32(0x04002000), U32(0x00042000), U32(0x04042000),
+            U32(0x00002002), U32(0x04002002), U32(0x00042002), U32(0x04042002),
+            U32(0x00000020), U32(0x04000020), U32(0x00040020), U32(0x04040020),
+            U32(0x00000022), U32(0x04000022), U32(0x00040022), U32(0x04040022),
+            U32(0x00002020), U32(0x04002020), U32(0x00042020), U32(0x04042020),
+            U32(0x00002022), U32(0x04002022), U32(0x00042022), U32(0x04042022),
+            U32(0x00000800), U32(0x04000800), U32(0x00040800), U32(0x04040800),
+            U32(0x00000802), U32(0x04000802), U32(0x00040802), U32(0x04040802),
+            U32(0x00002800), U32(0x04002800), U32(0x00042800), U32(0x04042800),
+            U32(0x00002802), U32(0x04002802), U32(0x00042802), U32(0x04042802),
+            U32(0x00000820), U32(0x04000820), U32(0x00040820), U32(0x04040820),
+            U32(0x00000822), U32(0x04000822), U32(0x00040822), U32(0x04040822),
+            U32(0x00002820), U32(0x04002820), U32(0x00042820), U32(0x04042820),
+            U32(0x00002822), U32(0x04002822), U32(0x00042822), U32(0x04042822),
+        ]
+
+    ]

packages/wakatime/packages/ntlm_auth/gss_channel_bindings.py

@@ -0,0 +1,67 @@
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+import struct
+
+"""
+    This is not the easiest structure to understand, ultimately this is a set structure
+    as defined by Microsoft. Channel Binding Tokens set the SHA256 hash of the server
+    certificate to the application_data field and then ultimately creates the MD5 hash
+    to include in the NTLM auth from there. This class is just designed to create the
+    bindings structure which is then used by compute_response.py to do the rest of the
+    work.
+
+    For more infor on how this works and how it is derived, this is a great link;
+    https://blogs.msdn.microsoft.com/openspecification/2013/03/26/ntlm-and-channel-binding-hash-aka-extended-protection-for-authentication/
+"""
+class GssChannelBindingsStruct(object):
+    INITIATOR_ADDTYPE = 'initiator_addtype'
+    INITIATOR_ADDRESS_LENGTH = 'initiator_address_length'
+    ACCEPTOR_ADDRTYPE = 'acceptor_addrtype'
+    ACCEPTOR_ADDRESS_LENGTH = 'acceptor_address_length'
+    APPLICATION_DATA_LENGTH = 'application_data_length'
+    INITIATOR_ADDRESS = 'initiator_address'
+    ACCEPTOR_ADDRESS = 'acceptor_address'
+    APPLICATION_DATA = 'application_data'
+
+    def __init__(self):
+        self.fields = {}
+        self.fields[self.INITIATOR_ADDTYPE] = 0
+        self.fields[self.INITIATOR_ADDRESS_LENGTH] = 0
+        self.fields[self.ACCEPTOR_ADDRTYPE] = 0
+        self.fields[self.ACCEPTOR_ADDRESS_LENGTH] = 0
+        self.fields[self.APPLICATION_DATA_LENGTH] = 0
+        self.fields[self.INITIATOR_ADDRESS] = b''
+        self.fields[self.ACCEPTOR_ADDRESS] = b''
+        self.fields[self.APPLICATION_DATA] = b''
+
+    def __setitem__(self, key, value):
+        self.fields[key] = value
+
+    def get_data(self):
+        # Set the lengths of each len field in case they have changed
+        self.fields[self.INITIATOR_ADDRESS_LENGTH] = len(self.fields[self.INITIATOR_ADDRESS])
+        self.fields[self.ACCEPTOR_ADDRESS_LENGTH] = len(self.fields[self.ACCEPTOR_ADDRESS])
+        self.fields[self.APPLICATION_DATA_LENGTH] = len(self.fields[self.APPLICATION_DATA])
+
+        # Add all the values together to create the gss_channel_bindings_struct
+        data = struct.pack('<L', self.fields[self.INITIATOR_ADDTYPE]) + \
+               struct.pack('<L', self.fields[self.INITIATOR_ADDRESS_LENGTH]) + \
+               self.fields[self.INITIATOR_ADDRESS] + \
+               struct.pack('<L', self.fields[self.ACCEPTOR_ADDRTYPE]) + \
+               struct.pack('<L', self.fields[self.ACCEPTOR_ADDRESS_LENGTH]) + \
+               self.fields[self.ACCEPTOR_ADDRESS] + \
+               struct.pack('<L', self.fields[self.APPLICATION_DATA_LENGTH]) + \
+               self.fields[self.APPLICATION_DATA]
+
+        return data

packages/wakatime/packages/ntlm_auth/messages.py

@@ -0,0 +1,359 @@
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+import hmac
+import os
+import struct
+from ntlm_auth.compute_response import ComputeResponse
+from ntlm_auth.constants import NegotiateFlags, MessageTypes, NTLM_SIGNATURE, AvFlags
+from ntlm_auth.rc4 import ARC4
+from ntlm_auth.target_info import TargetInfo
+
+class NegotiateMessage(object):
+    EXPECTED_BODY_LENGTH = 40
+
+    """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.1.1 NEGOTIATE_MESSAGE
+        The NEGOTIATE_MESSAGE defines an NTLM Negotiate message that is sent from the client to
+        the server. This message allows the client to specify its supported NTLM options to
+        the server.
+
+        :param negotiate_flags: A NEGOTIATE structure that contains a set of bit flags. These flags are the options the client supports
+        :param domain_name: The domain name of the user to authenticate with, default is None
+        :param workstation: The worksation of the client machine, default is None
+
+        Attributes:
+            signature: An 8-byte character array that MUST contain the ASCII string 'NTLMSSP\0'
+            message_type: A 32-bit unsigned integer that indicates the message type. This field must be set to 0x00000001
+            negotiate_flags: A NEGOTIATE structure that contains a set of bit flags. These flags are the options the client supports
+            version: Contains the windows version info of the client. It is used only debugging purposes and are only set when NTLMSSP_NEGOTIATE_VERSION flag is set
+            domain_name: A byte-array that contains the name of the client authentication domain that MUST Be encoded in the negotiated character set
+            workstation: A byte-array that contains the name of the client machine that MUST Be encoded in the negotiated character set
+    """
+    def __init__(self, negotiate_flags, domain_name, workstation):
+        self.signature = NTLM_SIGNATURE
+        self.message_type = struct.pack('<L', MessageTypes.NTLM_NEGOTIATE)
+
+        # Check if the domain_name value is set, if it is, make sure the negotiate_flag is also set
+        if domain_name is None:
+            self.domain_name = ''
+        else:
+            self.domain_name = domain_name
+            negotiate_flags |= NegotiateFlags.NTLMSSP_NEGOTIATE_OEM_DOMAIN_SUPPLIED
+
+        # Check if the workstation value is set, if it is, make sure the negotiate_flag is also set
+        if workstation is None:
+            self.workstation = ''
+        else:
+            self.workstation = workstation
+            negotiate_flags |= NegotiateFlags.NTLMSSP_NEGOTIATE_OEM_WORKSTATION_SUPPLIED
+
+        # Set the encoding flag to use OEM, remove UNICODE if set as it isn't support in this message
+        negotiate_flags -= NegotiateFlags.NTLMSSP_NEGOTIATE_UNICODE
+        negotiate_flags |= NegotiateFlags.NTLMSSP_NEGOTIATE_OEM
+        self.domain_name = self.domain_name.encode('ascii')
+        self.workstation = self.workstation.encode('ascii')
+
+        self.version = get_version(negotiate_flags)
+
+        self.negotiate_flags = struct.pack('<I', negotiate_flags)
+
+    def get_data(self):
+        payload_offset = self.EXPECTED_BODY_LENGTH
+
+        # DomainNameFields - 8 bytes
+        domain_name_len = struct.pack('<H', len(self.domain_name))
+        domain_name_max_len = struct.pack('<H', len(self.domain_name))
+        domain_name_buffer_offset = struct.pack('<I', payload_offset)
+        payload_offset += len(self.domain_name)
+
+        # WorkstationFields - 8 bytes
+        workstation_len = struct.pack('<H', len(self.workstation))
+        workstation_max_len = struct.pack('<H', len(self.workstation))
+        workstation_buffer_offset = struct.pack('<I', payload_offset)
+        payload_offset += len(self.workstation)
+
+        # Payload - variable length
+        payload = self.domain_name
+        payload += self.workstation
+
+        # Bring the header values together into 1 message
+        msg1 = self.signature
+        msg1 += self.message_type
+        msg1 += self.negotiate_flags
+        msg1 += domain_name_len
+        msg1 += domain_name_max_len
+        msg1 += domain_name_buffer_offset
+        msg1 += workstation_len
+        msg1 += workstation_max_len
+        msg1 += workstation_buffer_offset
+        msg1 += self.version
+
+        assert self.EXPECTED_BODY_LENGTH == len(msg1), "BODY_LENGTH: %d != msg1: %d" % (self.EXPECTED_BODY_LENGTH, len(msg1))
+
+        # Adding the payload data to the message
+        msg1 += payload
+        return msg1
+
+class ChallengeMessage(object):
+    """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.1.2 CHALLENGE_MESSAGE
+        The CHALLENGE_MESSAGE defines an NTLM challenge message that is sent from the server to
+        the client. The CHALLENGE_MESSAGE is used by the server to challenge the client to prove
+        its identity, For connection-oriented requests, the CHALLENGE_MESSAGE generated by the
+        server is in response to the NEGOTIATE_MESSAGE from the client.
+
+        :param msg2: The CHALLENGE_MESSAGE received from the server after sending our NEGOTIATE_MESSAGE. This has
+                        been decoded from a base64 string
+
+        Attributes
+            signature: An 8-byte character array that MUST contain the ASCII string 'NTLMSSP\0'
+            message_type: A 32-bit unsigned integer that indicates the message type. This field must be set to 0x00000002
+            negotiate_flags: A NEGOTIATE strucutre that contains a set of bit flags. The server sets flags to indicate options it supports
+            server_challenge: A 64-bit value that contains the NTLM challenge. The challenge is a 64-bit nonce. Used in the AuthenticateMessage message
+            reserved: An 8-byte array whose elements MUST be zero when sent and MUST be ignored on receipt
+            version: When NTLMSSP_NEGOTIATE_VERSION flag is set in negotiate_flags field which contains the windows version info. Used only for debugging purposes
+            target_name: When NTLMSSP_REQUEST_TARGET is set is a byte array that contains the name of the server authentication realm. In a domain environment this is the domain name not server name
+            target_info: When NTLMSSP_NEGOTIATE_TARGET_INFO is set is a byte array that contains a sequence of AV_PAIR structures (target_info.py)
+    """
+    def __init__(self, msg2):
+        self.data = msg2
+        # Setting the object values from the raw_challenge_message
+        self.signature = msg2[0:8]
+        self.message_type = struct.unpack("<I", msg2[8:12])[0]
+        self.negotiate_flags = struct.unpack("<I", msg2[20:24])[0]
+        self.server_challenge = msg2[24:32]
+        self.reserved = msg2[32:40]
+
+        if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_VERSION and self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY:
+            size = len(msg2)
+            self.version = struct.unpack("<q", msg2[48:56])[0]
+        else:
+            self.version = None
+
+        if self.negotiate_flags & NegotiateFlags.NTLMSSP_REQUEST_TARGET:
+            target_name_len = struct.unpack("<H", msg2[12:14])[0]
+            target_name_max_len = struct.unpack("<H", msg2[14:16])[0]
+            target_name_buffer_offset = struct.unpack("<I", msg2[16:20])[0]
+            self.target_name = msg2[target_name_buffer_offset:target_name_buffer_offset + target_name_len]
+        else:
+            self.target_name = None
+
+        if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_TARGET_INFO:
+            target_info_len = struct.unpack("<H", msg2[40:42])[0]
+            target_info_max_len = struct.unpack("<H", msg2[42:44])[0]
+            target_info_buffer_offset = struct.unpack("<I", msg2[44:48])[0]
+
+            target_info_raw = msg2[target_info_buffer_offset:target_info_buffer_offset + target_info_len]
+            self.target_info = TargetInfo(target_info_raw)
+        else:
+            self.target_info = None
+
+        # Verify initial integrity of the message, it matches what should be there
+        assert self.signature == NTLM_SIGNATURE
+        assert self.message_type == MessageTypes.NTLM_CHALLENGE
+
+    def get_data(self):
+        return self.data
+
+class AuthenticateMessage(object):
+    EXPECTED_BODY_LENGTH = 72
+    EXPECTED_BODY_LENGTH_WITH_MIC = 88
+
+    """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.1.3 AUTHENTICATE_MESSAGE
+        The AUTHENTICATE_MESSAGE defines an NTLM authenticate message that is sent from the
+        client to the server after the CHALLENGE_MESSAGE is processed by the client.
+
+        :param user_name: The user name of the user we are trying to authenticate with
+        :param password: The password of the user we are trying to authenticate with
+        :param domain_name: The domain name of the user account we are authenticated with, default is None
+        :param workstation: The workstation we are using to authenticate with, default is None
+        :param challenge_message: A ChallengeMessage object that was received from the server after the negotiate_message
+        :param ntlm_compatibility: The Lan Manager Compatibility Level, used to determine what NTLM auth version to use, see Ntlm in ntlm.py for more details
+        :param server_certificate_hash: The SHA256 hash string of the server certificate (DER encoded) NTLM is authenticating to. This is used to add
+                                          to the gss_channel_bindings_struct for Channel Binding Tokens support. If none is passed through then ntlm-auth
+                                          will not use Channel Binding Tokens when authenticating with the server which could cause issues if it is set to
+                                          only authenticate when these are present. This is only used for NTLMv2 authentication.
+
+        Message Attributes (Attributes not used to compute the message structure):
+            signature: An 8-byte character array that MUST contain the ASCII string 'NTLMSSP\0'
+            message_type: A 32-bit unsigned integer that indicates the message type. This field must be set to 0x00000003
+            negotiate_flags: A NEGOTIATE strucutre that contains a set of bit flags. These flags are the choices the client has made from the CHALLENGE_MESSAGE options
+            version: Contains the windows version info of the client. It is used only debugging purposes and are only set when NTLMSSP_NEGOTIATE_VERSION flag is set
+            mic: The message integrity for the NEGOTIATE_MESSAGE, CHALLENGE_MESSAGE and AUTHENTICATE_MESSAGE
+            lm_challenge_response: An LM_RESPONSE of LMv2_RESPONSE structure that contains the computed LM response to the challenge
+            nt_challenge_response: An NTLM_RESPONSE or NTLMv2_RESPONSE structure that contains the computed NT response to the challenge
+            domain_name: The domain or computer name hosting the user account, MUST be encoded in the negotiated character set
+            user_name: The name of the user to be authenticated, MUST be encoded in the negotiated character set
+            workstation: The name of the computer to which the user is logged on, MUST Be encoded in the negotiated character set
+            encrypted_random_session_key: The client's encrypted random session key
+
+        Non-Message Attributes (Attributes not used to compute the message structure):
+            exported_session_key: A randomly generated session key based on other keys, used to derive the SIGNKEY and SEALKEY
+            target_info: The AV_PAIR structure used in the nt response calculation
+    """
+    def __init__(self, user_name, password, domain_name, workstation, challenge_message, ntlm_compatibility, server_certificate_hash):
+        self.signature = NTLM_SIGNATURE
+        self.message_type = struct.pack('<L', MessageTypes.NTLM_AUTHENTICATE)
+        self.negotiate_flags = challenge_message.negotiate_flags
+        self.version = get_version(self.negotiate_flags)
+        self.mic = None
+
+        if domain_name is None:
+            self.domain_name = ''
+        else:
+            self.domain_name = domain_name
+
+        if workstation is None:
+            self.workstation = ''
+        else:
+            self.workstation = workstation
+
+        if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_UNICODE:
+            self.negotiate_flags -= NegotiateFlags.NTLMSSP_NEGOTIATE_OEM
+            encoding_value = 'utf-16-le'
+        else:
+            encoding_value = 'ascii'
+
+        self.domain_name = self.domain_name.encode(encoding_value)
+        self.user_name = user_name.encode(encoding_value)
+        self.workstation = self.workstation.encode(encoding_value)
+
+        compute_response = ComputeResponse(user_name, password, domain_name, challenge_message,
+                                           ntlm_compatibility)
+
+        self.lm_challenge_response = compute_response.get_lm_challenge_response()
+        self.nt_challenge_response, key_exchange_key, target_info = compute_response.get_nt_challenge_response(
+            self.lm_challenge_response, server_certificate_hash)
+        self.target_info = target_info
+
+        if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_KEY_EXCH:
+            self.exported_session_key = get_random_export_session_key()
+
+            rc4_handle = ARC4(key_exchange_key)
+            self.encrypted_random_session_key = rc4_handle.update(self.exported_session_key)
+        else:
+            self.exported_session_key = key_exchange_key
+            self.encrypted_random_session_key = b''
+
+        self.negotiate_flags = struct.pack('<I', self.negotiate_flags)
+
+    def get_data(self):
+        if self.mic is None:
+            mic = b''
+            expected_body_length = self.EXPECTED_BODY_LENGTH
+        else:
+            mic = self.mic
+            expected_body_length = self.EXPECTED_BODY_LENGTH_WITH_MIC
+
+        payload_offset = expected_body_length
+
+        # DomainNameFields - 8 bytes
+        domain_name_len = struct.pack('<H', len(self.domain_name))
+        domain_name_max_len = struct.pack('<H', len(self.domain_name))
+        domain_name_buffer_offset = struct.pack('<I', payload_offset)
+        payload_offset += len(self.domain_name)
+
+        # UserNameFields - 8 bytes
+        user_name_len = struct.pack('<H', len(self.user_name))
+        user_name_max_len = struct.pack('<H', len(self.user_name))
+        user_name_buffer_offset = struct.pack('<I', payload_offset)
+        payload_offset += len(self.user_name)
+
+        # WorkstatonFields - 8 bytes
+        workstation_len = struct.pack('<H', len(self.workstation))
+        workstation_max_len = struct.pack('<H', len(self.workstation))
+        workstation_buffer_offset = struct.pack('<I', payload_offset)
+        payload_offset += len(self.workstation)
+
+        # LmChallengeResponseFields - 8 bytes
+        lm_challenge_response_len = struct.pack('<H', len(self.lm_challenge_response))
+        lm_challenge_response_max_len = struct.pack('<H', len(self.lm_challenge_response))
+        lm_challenge_response_buffer_offset = struct.pack('<I', payload_offset)
+        payload_offset += len(self.lm_challenge_response)
+
+        # NtChallengeResponseFields - 8 bytes
+        nt_challenge_response_len = struct.pack('<H', len(self.nt_challenge_response))
+        nt_challenge_response_max_len = struct.pack('<H', len(self.nt_challenge_response))
+        nt_challenge_response_buffer_offset = struct.pack('<I', payload_offset)
+        payload_offset += len(self.nt_challenge_response)
+
+        # EncryptedRandomSessionKeyFields - 8 bytes
+        encrypted_random_session_key_len = struct.pack('<H', len(self.encrypted_random_session_key))
+        encrypted_random_session_key_max_len = struct.pack('<H', len(self.encrypted_random_session_key))
+        encrypted_random_session_key_buffer_offset = struct.pack('<I', payload_offset)
+        payload_offset += len(self.encrypted_random_session_key)
+
+        # Payload - variable length
+        payload = self.domain_name
+        payload += self.user_name
+        payload += self.workstation
+        payload += self.lm_challenge_response
+        payload += self.nt_challenge_response
+        payload += self.encrypted_random_session_key
+
+        msg3 = self.signature
+        msg3 += self.message_type
+        msg3 += lm_challenge_response_len + lm_challenge_response_max_len + lm_challenge_response_buffer_offset
+        msg3 += nt_challenge_response_len + nt_challenge_response_max_len + nt_challenge_response_buffer_offset
+        msg3 += domain_name_len + domain_name_max_len + domain_name_buffer_offset
+        msg3 += user_name_len + user_name_max_len + user_name_buffer_offset
+        msg3 += workstation_len + workstation_max_len + workstation_buffer_offset
+        msg3 += encrypted_random_session_key_len + encrypted_random_session_key_max_len + encrypted_random_session_key_buffer_offset
+        msg3 += self.negotiate_flags
+        msg3 += self.version
+        msg3 += mic
+
+        # Adding the payload data to the message
+        msg3 += payload
+
+        return msg3
+
+    def add_mic(self, negotiate_message, challenge_message):
+        if self.target_info is not None:
+            av_flags = self.target_info[TargetInfo.MSV_AV_FLAGS]
+
+            if av_flags is not None and av_flags[1] == struct.pack("<L", AvFlags.MIC_PROVIDED):
+                self.mic = struct.pack("<IIII", 0, 0, 0, 0)
+                negotiate_data = negotiate_message.get_data()
+                challenge_data = challenge_message.get_data()
+                authenticate_data = self.get_data()
+
+                mic = hmac.new(self.exported_session_key,
+                               (negotiate_data + challenge_data + authenticate_data)).digest()
+                self.mic = mic
+
+def get_version(negotiate_flags):
+    # Check the negotiate_flag version is set, if it is make sure the version info is added to the data
+    if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_VERSION:
+        # TODO: Get the major and minor version of Windows instead of using default values
+        product_major_version = struct.pack('<B', 6)
+        product_minor_version = struct.pack('<B', 1)
+        product_build = struct.pack('<H', 7601)
+        version_reserved = b'\0' * 3
+        ntlm_revision_current = struct.pack('<B', 15)
+        version = product_major_version + product_minor_version + product_build + version_reserved + ntlm_revision_current
+    else:
+        version = b'\0' * 8
+
+    return version
+
+def get_random_export_session_key():
+    return os.urandom(16)

packages/wakatime/packages/ntlm_auth/ntlm.py

@@ -0,0 +1,146 @@
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+import base64
+import socket
+import struct
+from ntlm_auth.constants import NegotiateFlags
+from ntlm_auth.messages import NegotiateMessage, ChallengeMessage, AuthenticateMessage
+from ntlm_auth.session_security import SessionSecurity
+
+
+"""
+utility functions for Microsoft NTLM authentication
+
+References:
+[MS-NLMP]: NT LAN Manager (NTLM) Authentication Protocol Specification
+http://download.microsoft.com/download/a/e/6/ae6e4142-aa58-45c6-8dcf-a657e5900cd3/%5BMS-NLMP%5D.pdf
+
+[MS-NTHT]: NTLM Over HTTP Protocol Specification
+http://download.microsoft.com/download/a/e/6/ae6e4142-aa58-45c6-8dcf-a657e5900cd3/%5BMS-NTHT%5D.pdf
+
+Cntlm Authentication Proxy
+http://cntlm.awk.cz/
+
+NTLM Authorization Proxy Server
+http://sourceforge.net/projects/ntlmaps/
+
+Optimized Attack for NTLM2 Session Response
+http://www.blackhat.com/presentations/bh-asia-04/bh-jp-04-pdfs/bh-jp-04-seki.pdf
+"""
+
+class Ntlm(object):
+    """
+    Initialises the NTLM context to use when sending and receiving messages to and from the server. You should be
+    using this object as it supports NTLMv2 authenticate and it easier to use than before. It also brings in the
+    ability to use signing and sealing with session_security and generate a MIC structure.
+
+    :param ntlm_compatibility: The Lan Manager Compatibility Level to use withe the auth message - Default 3
+                                This is set by an Administrator in the registry key
+                                'HKLM\SYSTEM\CurrentControlSet\Control\Lsa\LmCompatibilityLevel'
+                                The values correspond to the following;
+                                    0 : LM and NTLMv1
+                                    1 : LM, NTLMv1 and NTLMv1 with Extended Session Security
+                                    2 : NTLMv1 and NTLMv1 with Extended Session Security
+                                    3-5 : NTLMv2 Only
+                                Note: Values 3 to 5 are no different as the client supports the same types
+
+    Attributes:
+        negotiate_flags: A NEGOTIATE structure that contains a set of bit flags. These flags are the options the client supports and are sent in the negotiate_message
+        ntlm_compatibility: The Lan Manager Compatibility Level, same as the input if supplied
+        negotiate_message: A NegotiateMessage object that is sent to the server
+        challenge_message: A ChallengeMessage object that has been created from the server response
+        authenticate_message: An AuthenticateMessage object that is sent to the server based on the ChallengeMessage
+        session_security: A SessionSecurity structure that can be used to sign and seal messages sent after the authentication challenge
+    """
+    def __init__(self, ntlm_compatibility=3):
+        self.ntlm_compatibility = ntlm_compatibility
+
+        # Setting up our flags so the challenge message returns the target info block if supported
+        self.negotiate_flags = NegotiateFlags.NTLMSSP_NEGOTIATE_TARGET_INFO | \
+                               NegotiateFlags.NTLMSSP_NEGOTIATE_128 | \
+                               NegotiateFlags.NTLMSSP_NEGOTIATE_56 | \
+                               NegotiateFlags.NTLMSSP_NEGOTIATE_UNICODE | \
+                               NegotiateFlags.NTLMSSP_NEGOTIATE_VERSION | \
+                               NegotiateFlags.NTLMSSP_NEGOTIATE_KEY_EXCH | \
+                               NegotiateFlags.NTLMSSP_NEGOTIATE_ALWAYS_SIGN | \
+                               NegotiateFlags.NTLMSSP_NEGOTIATE_SIGN | \
+                               NegotiateFlags.NTLMSSP_NEGOTIATE_SEAL
+
+        # Setting the message types based on the ntlm_compatibility level
+        self._set_ntlm_compatibility_flags(self.ntlm_compatibility)
+
+        self.negotiate_message = None
+        self.challenge_message = None
+        self.authenticate_message = None
+        self.session_security = None
+
+
+    def create_negotiate_message(self, domain_name=None, workstation=None):
+        """
+        Create an NTLM NEGOTIATE_MESSAGE
+
+        :param domain_name: The domain name of the user account we are authenticating with, default is None
+        :param worksation: The workstation we are using to authenticate with, default is None
+        :return: A base64 encoded string of the NEGOTIATE_MESSAGE
+        """
+        self.negotiate_message = NegotiateMessage(self.negotiate_flags, domain_name, workstation)
+
+        return base64.b64encode(self.negotiate_message.get_data())
+
+    def parse_challenge_message(self, msg2):
+        """
+        Parse the NTLM CHALLENGE_MESSAGE from the server and add it to the Ntlm context fields
+
+        :param msg2: A base64 encoded string of the CHALLENGE_MESSAGE
+        """
+        msg2 = base64.b64decode(msg2)
+        self.challenge_message = ChallengeMessage(msg2)
+
+    def create_authenticate_message(self, user_name, password, domain_name=None, workstation=None, server_certificate_hash=None):
+        """
+        Create an NTLM AUTHENTICATE_MESSAGE based on the Ntlm context and the previous messages sent and received
+
+        :param user_name: The user name of the user we are trying to authenticate with
+        :param password: The password of the user we are trying to authenticate with
+        :param domain_name: The domain name of the user account we are authenticated with, default is None
+        :param workstation: The workstation we are using to authenticate with, default is None
+        :param server_certificate_hash: The SHA256 hash string of the server certificate (DER encoded) NTLM is authenticating to. Used for Channel
+                                        Binding Tokens. If nothing is supplied then the CBT hash will not be sent. See messages.py AuthenticateMessage
+                                        for more details
+        :return: A base64 encoded string of the AUTHENTICATE_MESSAGE
+        """
+        self.authenticate_message = AuthenticateMessage(user_name, password, domain_name, workstation,
+                                                        self.challenge_message, self.ntlm_compatibility,
+                                                        server_certificate_hash)
+        self.authenticate_message.add_mic(self.negotiate_message, self.challenge_message)
+
+        # Setups up the session_security context used to sign and seal messages if wanted
+        if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_SEAL or self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_SIGN:
+            self.session_security = SessionSecurity(struct.unpack("<I", self.authenticate_message.negotiate_flags)[0],
+                                                    self.authenticate_message.exported_session_key)
+
+        return base64.b64encode(self.authenticate_message.get_data())
+
+    def _set_ntlm_compatibility_flags(self, ntlm_compatibility):
+        if (ntlm_compatibility >= 0) and (ntlm_compatibility <= 5):
+            if ntlm_compatibility == 0:
+                self.negotiate_flags |= NegotiateFlags.NTLMSSP_NEGOTIATE_NTLM | \
+                                        NegotiateFlags.NTLMSSP_NEGOTIATE_LM_KEY
+            elif ntlm_compatibility == 1:
+                self.negotiate_flags |= NegotiateFlags.NTLMSSP_NEGOTIATE_NTLM | \
+                                        NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY
+            else:
+                self.negotiate_flags |= NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY
+        else:
+            raise Exception("Unknown ntlm_compatibility level - expecting value between 0 and 5")

packages/wakatime/packages/ntlm_auth/rc4.py

@@ -0,0 +1,51 @@
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+class ARC4(object):
+    state = None
+    i = 0
+    j = 0
+
+    def __init__(self, key):
+        # Split up the key into a list
+        if isinstance(key, str):
+            key = [ord(c) for c in key]
+        else:
+            key = [c for c in key]
+
+        #Key-scheduling algorithm (KSA)
+        self.state = [n for n in range(256)]
+        j = 0
+        for i in range(256):
+            j = (j + self.state[i] + key[i % len(key)]) % 256
+            self.state[i], self.state[j] = self.state[j], self.state[i]
+
+    def update(self, value):
+        chars = []
+        random_gen = self._random_generator()
+        for char in value:
+            if isinstance(value, str):
+                byte = ord(char)
+            else:
+                byte = char
+            updated_byte = byte ^ next(random_gen)
+            chars.append(updated_byte)
+        return bytes(bytearray(chars))
+
+    def _random_generator(self):
+        #Pseudo-Random Generation Algorithm (PRGA)
+        while True:
+            self.i = (self.i + 1) % 256
+            self.j = (self.j + self.state[self.i]) % 256
+            self.state[self.i], self.state[self.j] = self.state[self.j], self.state[self.i]
+            yield self.state[(self.state[self.i] + self.state[self.j]) % 256]

packages/wakatime/packages/ntlm_auth/session_security.py

@@ -0,0 +1,250 @@
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+import binascii
+import hmac
+import struct
+import ntlm_auth.compute_keys as compkeys
+from ntlm_auth.constants import NegotiateFlags, SignSealConstants
+from ntlm_auth.rc4 import ARC4
+
+
+class _NtlmMessageSignature1(object):
+    EXPECTED_BODY_LENGTH = 16
+
+    """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.2.9.1 NTLMSSP_MESSAGE_SIGNATURE
+        This version of the NTLMSSP_MESSAGE_SIGNATURE structure MUST be used when the
+        NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY flag is not negotiated.
+
+        :param random_pad: A 4-byte array that contains the random pad for the emssage
+        :param checksum: A 4-byte array that contains the checksum for the message
+        :param seq_num: A 32-bit unsigned integer that contains the NTLM sequence number for this application message
+    """
+    def __init__(self, random_pad, checksum, seq_num):
+        self.version = struct.pack("<I", 1)
+        self.random_pad = random_pad
+        self.checksum = checksum
+        self.seq_num = seq_num
+
+    def get_data(self):
+        signature = self.version
+        signature += self.random_pad
+        signature += self.checksum
+        signature += self.seq_num
+
+        assert self.EXPECTED_BODY_LENGTH == len(signature), "BODY_LENGTH: %d != signature: %d" % (
+        self.EXPECTED_BODY_LENGTH, len(signature))
+
+        return signature
+
+class _NtlmMessageSignature2(object):
+    EXPECTED_BODY_LENGTH = 16
+
+    """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.2.9.2 NTLMSSP_MESSAGE_SIGNATURE for Extended Session Security
+        This version of the NTLMSSP_MESSAGE_SIGNATURE structure MUST be used when the
+        NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY flag is negotiated
+
+        :param checksum: An 8-byte array that contains the checksum for the message
+        :param seq_num: A 32-bit unsigned integer that contains the NTLM sequence number for this application message
+    """
+
+    def __init__(self, checksum, seq_num):
+        self.version = struct.pack("<I", 1)
+        self.checksum = checksum
+        self.seq_num = seq_num
+
+    def get_data(self):
+        signature = self.version
+        signature += self.checksum
+        signature += self.seq_num
+
+        assert self.EXPECTED_BODY_LENGTH == len(signature), "BODY_LENGTH: %d != signature: %d" % (
+            self.EXPECTED_BODY_LENGTH, len(signature))
+
+        return signature
+
+class SessionSecurity(object):
+    """
+    Initialises a security session context that can be used by libraries that call ntlm-auth to sign and seal
+    messages send to the server as well as verify and unseal messages that have been received from the server.
+    This is similar to the GSS_Wrap functions specified in the MS-NLMP document which does the same task.
+
+    :param negotiate_flags: The negotiate flag structure that has been negotiated with the server
+    :param exported_session_key: A 128-bit session key used to derive signing and sealing keys
+    :param source: The source of the message, only used in test scenarios when testing out a server sealing and unsealing
+    """
+    def __init__(self, negotiate_flags, exported_session_key, source="client"):
+        self.negotiate_flags = negotiate_flags
+        self.outgoing_seq_num = 0
+        self.incoming_seq_num = 0
+
+        client_sealing_key = compkeys.get_seal_key(self.negotiate_flags, exported_session_key, SignSealConstants.CLIENT_SEALING)
+        server_sealing_key = compkeys.get_seal_key(self.negotiate_flags, exported_session_key, SignSealConstants.SERVER_SEALING)
+
+        if source == "client":
+            self.outgoing_signing_key = compkeys.get_sign_key(exported_session_key, SignSealConstants.CLIENT_SIGNING)
+            self.incoming_signing_key = compkeys.get_sign_key(exported_session_key, SignSealConstants.SERVER_SIGNING)
+            self.outgoing_handle = ARC4(client_sealing_key)
+            self.incoming_handle = ARC4(server_sealing_key)
+        elif source == "server":
+            self.outgoing_signing_key = compkeys.get_sign_key(exported_session_key, SignSealConstants.SERVER_SIGNING)
+            self.incoming_signing_key = compkeys.get_sign_key(exported_session_key, SignSealConstants.CLIENT_SIGNING)
+            self.outgoing_handle = ARC4(server_sealing_key)
+            self.incoming_handle = ARC4(client_sealing_key)
+        else:
+            raise Exception("Invalid source parameter %s, must be client or server" % source)
+
+    def wrap(self, message):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        3.4.6 GSS_WrapEx()
+        Emulates the GSS_Wrap() implementation to sign and seal messages if the correct flags
+        are set.
+
+        @param message: The message data that will be wrapped
+        @return message: The message that has been sealed if flags are set
+        @return signature: The signature of the message, None if flags are not set
+        """
+        if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_SEAL:
+            encrypted_message = self._seal_message(message)
+            signature = self._get_signature(message)
+            message = encrypted_message
+
+        elif self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_SIGN:
+            signature = self._get_signature(message)
+        else:
+            signature = None
+
+        return message, signature
+
+    def unwrap(self, message, signature):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        3.4.7 GSS_UnwrapEx()
+        Emulates the GSS_Unwrap() implementation to unseal messages and verify the signature
+        sent matches what has been computed locally. Will throw an Exception if the signature
+        doesn't match
+
+        @param message: The message data received from the server
+        @param signature: The signature of the message
+        @return message: The message that has been unsealed if flags are set
+        """
+        if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_SEAL:
+            message = self._unseal_message(message)
+            self._verify_signature(message, signature)
+
+        elif self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_SIGN:
+            self._verify_signature(message, signature)
+
+        return message
+
+    def _seal_message(self, message):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        3.4.3 Message Confidentiality
+        Will generate an encrypted message using RC4 based on the ClientSealingKey
+
+        @param message: The message to be sealed (encrypted)
+        @return encrypted_message: The encrypted message
+        """
+        encrypted_message = self.outgoing_handle.update(message)
+        return encrypted_message
+
+    def _unseal_message(self, message):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        3.4.3 Message Confidentiality
+        Will generate a dencrypted message using RC4 based on the ServerSealingKey
+
+        @param message: The message to be unsealed (dencrypted)
+        @return decrypted_message: The decrypted message
+        """
+        decrypted_message = self.incoming_handle.update(message)
+        return decrypted_message
+
+    def _get_signature(self, message):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        3.4.4 Message Signature Functions
+        Will create the signature based on the message to send to the server. Depending on the negotiate_flags
+        set this could either be an NTLMv1 signature or NTLMv2 with Extended Session Security signature.
+
+        @param message: The message data that will be signed
+        @return signature: Either _NtlmMessageSignature1 or _NtlmMessageSignature2 depending on the flags set
+        """
+        signature = calc_signature(message, self.negotiate_flags, self.outgoing_signing_key, self.outgoing_seq_num, self.outgoing_handle)
+        self.outgoing_seq_num += 1
+
+        return signature.get_data()
+
+    def _verify_signature(self, message, signature):
+        """
+        Will verify that the signature received from the server matches up with the expected signature
+        computed locally. Will throw an exception if they do not match
+
+        @param message: The message data that is received from the server
+        @param signature: The signature of the message received from the server
+        """
+        if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY:
+            actual_checksum = signature[4:12]
+            actual_seq_num = struct.unpack("<I", signature[12:16])[0]
+        else:
+            actual_checksum = signature[8:12]
+            actual_seq_num = struct.unpack("<I", signature[12:16])[0]
+
+        expected_signature = calc_signature(message, self.negotiate_flags, self.incoming_signing_key, self.incoming_seq_num, self.incoming_handle)
+        expected_checksum = expected_signature.checksum
+        expected_seq_num = struct.unpack("<I", expected_signature.seq_num)[0]
+
+        if actual_checksum != expected_checksum:
+            raise Exception("The signature checksum does not match, message has been altered")
+
+        if actual_seq_num != expected_seq_num:
+            raise Exception("The signature sequence number does not match up, message not received in the correct sequence")
+
+        self.incoming_seq_num += 1
+
+
+def calc_signature(message, negotiate_flags, signing_key, seq_num, handle):
+    seq_num = struct.pack("<I", seq_num)
+    if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY:
+        checksum_hmac = hmac.new(signing_key, seq_num + message)
+        if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_KEY_EXCH:
+            checksum = handle.update(checksum_hmac.digest()[:8])
+        else:
+            checksum = checksum_hmac.digest()[:8]
+
+        signature = _NtlmMessageSignature2(checksum, seq_num)
+
+    else:
+        message_crc = binascii.crc32(message) % (1 << 32)
+        checksum = struct.pack("<I", message_crc)
+        random_pad = handle.update(struct.pack("<I", 0))
+        checksum = handle.update(checksum)
+        seq_num = handle.update(seq_num)
+        random_pad = struct.pack("<I", 0)
+
+        signature = _NtlmMessageSignature1(random_pad, checksum, seq_num)
+
+    return signature

packages/wakatime/packages/ntlm_auth/target_info.py

@@ -0,0 +1,68 @@
+"""
+    Original Author: Ian Clegg
+    Project: ntlmlib
+    URL: https://github.com/ianclegg/ntlmlib
+    License: Apache 2.0 License
+    Notes: Most of this code has been copied from the messages.py in the ntlmlib repo.
+    Some minor changes such as the name of the AV Pairs and extra comments have been added.
+"""
+
+import struct
+try:
+    from collections import OrderedDict
+except ImportError:
+    from ordereddict import OrderedDict
+
+class TargetInfo(object):
+    MSV_AV_EOL                  = 0x00
+    MSV_AV_NB_COMPUTER_NAME     = 0x01
+    MSV_AV_NB_DOMAIN_NAME       = 0x02
+    MSV_AV_DNS_COMPUTER_NAME    = 0x03
+    MSV_AV_DNS_DOMAIN_NAME      = 0x04
+    MSV_AV_DNS_TREE_NAME        = 0x05
+    MSV_AV_FLAGS                = 0x06
+    MSV_AV_TIMESTAMP            = 0x07
+    MSV_AV_SINGLE_HOST          = 0x08
+    MSV_AV_TARGET_NAME          = 0x09
+    MSV_AV_CHANNEL_BINDINGS     = 0x0a
+
+    def __init__(self, data=None):
+        self.fields = OrderedDict()
+        if data is not None:
+            self.from_string(data)
+
+    def __setitem__(self, key, value):
+        self.fields[key] = (len(value), value)
+
+    def __getitem__(self, key):
+        if key in self.fields:
+           return self.fields[key]
+        return None
+
+    def __delitem__(self, key):
+        del self.fields[key]
+
+    def from_string(self, data):
+        attribute_type = 0xff
+        while attribute_type is not TargetInfo.MSV_AV_EOL:
+            # Parse the Attribute Value pair from the structure
+            attribute_type = struct.unpack('<H', data[:struct.calcsize('<H')])[0]
+            data = data[struct.calcsize('<H'):]
+            length = struct.unpack('<H', data[:struct.calcsize('<H')])[0]
+            data = data[struct.calcsize('<H'):]
+            # Add a new field to the object for the parse attribute value
+            self.fields[attribute_type] = (length, data[:length])
+            data = data[length:]
+
+    def get_data(self):
+        if TargetInfo.MSV_AV_EOL in self.fields:
+            del self.fields[TargetInfo.MSV_AV_EOL]
+
+        data = b''
+        for i in self.fields.keys():
+            data += struct.pack('<HH', i, self[i][0])
+            data += self[i][1]
+
+        # end with a NTLMSSP_AV_EOL
+        data += struct.pack('<HH', TargetInfo.MSV_AV_EOL, 0)
+        return data

packages/wakatime/packages/pygments/__init__.py

@@ -22,20 +22,18 @@
     .. _Pygments tip:
        http://bitbucket.org/birkenfeld/pygments-main/get/tip.zip#egg=Pygments-dev
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
-
-__version__ = '2.0.1'
-__docformat__ = 'restructuredtext'
-
-__all__ = ['lex', 'format', 'highlight']
-
-
 import sys
 
 from pygments.util import StringIO, BytesIO
 
+__version__ = '2.2.0'
+__docformat__ = 'restructuredtext'
+
+__all__ = ['lex', 'format', 'highlight']
+
 
 def lex(code, lexer):
     """
@@ -44,14 +42,15 @@ def lex(code, lexer):
     try:
         return lexer.get_tokens(code)
     except TypeError as err:
-        if isinstance(err.args[0], str) and \
-           'unbound method get_tokens' in err.args[0]:
+        if (isinstance(err.args[0], str) and
+            ('unbound method get_tokens' in err.args[0] or
+             'missing 1 required positional argument' in err.args[0])):
             raise TypeError('lex() argument must be a lexer instance, '
                             'not a class')
         raise
 
 
-def format(tokens, formatter, outfile=None):
+def format(tokens, formatter, outfile=None):  # pylint: disable=redefined-builtin
     """
     Format a tokenlist ``tokens`` with the formatter ``formatter``.
 
@@ -61,15 +60,15 @@ def format(tokens, formatter, outfile=None):
     """
     try:
         if not outfile:
-            #print formatter, 'using', formatter.encoding
-            realoutfile = formatter.encoding and BytesIO() or StringIO()
+            realoutfile = getattr(formatter, 'encoding', None) and BytesIO() or StringIO()
             formatter.format(tokens, realoutfile)
             return realoutfile.getvalue()
         else:
             formatter.format(tokens, outfile)
     except TypeError as err:
-        if isinstance(err.args[0], str) and \
-           'unbound method format' in err.args[0]:
+        if (isinstance(err.args[0], str) and
+            ('unbound method format' in err.args[0] or
+             'missing 1 required positional argument' in err.args[0])):
             raise TypeError('format() argument must be a formatter instance, '
                             'not a class')
         raise
@@ -86,6 +85,6 @@ def highlight(code, lexer, formatter, outfile=None):
     return format(lex(code, lexer), formatter, outfile)
 
 
-if __name__ == '__main__':
+if __name__ == '__main__':  # pragma: no cover
     from pygments.cmdline import main
     sys.exit(main(sys.argv))

packages/wakatime/packages/pygments/cmdline.py

@@ -5,12 +5,13 @@
 
     Command line interface.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 from __future__ import print_function
 
+import os
 import sys
 import getopt
 from textwrap import dedent
@@ -19,18 +20,20 @@ from pygments import __version__, highlight
 from pygments.util import ClassNotFound, OptionError, docstring_headline, \
     guess_decode, guess_decode_from_terminal, terminal_encoding
 from pygments.lexers import get_all_lexers, get_lexer_by_name, guess_lexer, \
-    get_lexer_for_filename, find_lexer_class, TextLexer
+    load_lexer_from_file, get_lexer_for_filename, find_lexer_class_for_filename
+from pygments.lexers.special import TextLexer
 from pygments.formatters.latex import LatexEmbeddedLexer, LatexFormatter
 from pygments.formatters import get_all_formatters, get_formatter_by_name, \
-    get_formatter_for_filename, find_formatter_class, \
-    TerminalFormatter  # pylint:disable-msg=E0611
+    load_formatter_from_file, get_formatter_for_filename, find_formatter_class
+from pygments.formatters.terminal import TerminalFormatter
+from pygments.formatters.terminal256 import Terminal256Formatter
 from pygments.filters import get_all_filters, find_filter_class
 from pygments.styles import get_all_styles, get_style_by_name
 
 
 USAGE = """\
 Usage: %s [-l <lexer> | -g] [-F <filter>[:<options>]] [-f <formatter>]
-          [-O <options>] [-P <option=value>] [-s] [-o <outfile>] [<infile>]
+          [-O <options>] [-P <option=value>] [-s] [-v] [-x] [-o <outfile>] [<infile>]
 
        %s -S <style> -f <formatter> [-a <arg>] [-O <options>] [-P <option=value>]
        %s -L [<which> ...]
@@ -56,6 +59,14 @@ Likewise, <formatter> is a formatter name, and will be guessed from
 the extension of the output file name. If no output file is given,
 the terminal formatter will be used by default.
 
+The additional option -x allows custom lexers and formatters to be
+loaded from a .py file relative to the current working directory. For
+example, ``-l ./customlexer.py -x``. By default, this option expects a
+file with a class named CustomLexer or CustomFormatter; you can also
+specify your own class name with a colon (``-l ./lexer.py:MyLexer``).
+Users should be very careful not to use this option with untrusted files,
+because it will import and run them.
+
 With the -O option, you can give the lexer and formatter a comma-
 separated list of options, e.g. ``-O bg=light,python=cool``.
 
@@ -90,6 +101,9 @@ waiting to process the entire file.  This only works for stdin, and
 is intended for streaming input such as you get from 'tail -f'.
 Example usage: "tail -f sql.log | pygmentize -s -l sql"
 
+The -v option prints a detailed traceback on unhandled exceptions,
+which is useful for debugging and bug reports.
+
 The -h option prints this help.
 The -V option prints the package version.
 """
@@ -100,7 +114,7 @@ def _parse_options(o_strs):
     if not o_strs:
         return opts
     for o_str in o_strs:
-        if not o_str:
+        if not o_str.strip():
             continue
         o_args = o_str.split(',')
         for o_arg in o_args:
@@ -132,7 +146,7 @@ def _parse_filters(f_strs):
 def _print_help(what, name):
     try:
         if what == 'lexer':
-            cls = find_lexer_class(name)
+            cls = get_lexer_by_name(name)
             print("Help on the %s lexer:" % cls.name)
             print(dedent(cls.__doc__))
         elif what == 'formatter':
@@ -143,8 +157,10 @@ def _print_help(what, name):
             cls = find_filter_class(name)
             print("Help on the %s filter:" % name)
             print(dedent(cls.__doc__))
-    except AttributeError:
+        return 0
+    except (AttributeError, ValueError):
         print("%s not found!" % what, file=sys.stderr)
+        return 1
 
 
 def _print_list(what):
@@ -198,19 +214,7 @@ def _print_list(what):
             print("    %s" % docstring_headline(cls))
 
 
-def main(args=sys.argv):
-    """
-    Main command line entry point.
-    """
-    # pylint: disable-msg=R0911,R0912,R0915
-
-    usage = USAGE % ((args[0],) * 6)
-
-    try:
-        popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:hVHgs")
-    except getopt.GetoptError:
-        print(usage, file=sys.stderr)
-        return 2
+def main_inner(popts, args, usage):
     opts = {}
     O_opts = []
     P_opts = []
@@ -229,7 +233,7 @@ def main(args=sys.argv):
         return 0
 
     if opts.pop('-V', None) is not None:
-        print('Pygments version %s, (c) 2006-2014 by Georg Brandl.' % __version__)
+        print('Pygments version %s, (c) 2006-2017 by Georg Brandl.' % __version__)
         return 0
 
     # handle ``pygmentize -L``
@@ -254,13 +258,12 @@ def main(args=sys.argv):
             print(usage, file=sys.stderr)
             return 2
 
-        what, name = args
+        what, name = args  # pylint: disable=unbalanced-tuple-unpacking
         if what not in ('lexer', 'formatter', 'filter'):
             print(usage, file=sys.stderr)
             return 2
 
-        _print_help(what, name)
-        return 0
+        return _print_help(what, name)
 
     # parse -O options
     parsed_opts = _parse_options(O_opts)
@@ -277,19 +280,15 @@ def main(args=sys.argv):
     opts.pop('-P', None)
 
     # encodings
-    inencoding  = parsed_opts.get('inencoding', parsed_opts.get('encoding'))
+    inencoding = parsed_opts.get('inencoding', parsed_opts.get('encoding'))
     outencoding = parsed_opts.get('outencoding', parsed_opts.get('encoding'))
 
     # handle ``pygmentize -N``
     infn = opts.pop('-N', None)
     if infn is not None:
-        try:
-            lexer = get_lexer_for_filename(infn, **parsed_opts)
-        except ClassNotFound as err:
-            lexer = TextLexer()
-        except OptionError as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
+        lexer = find_lexer_class_for_filename(infn)
+        if lexer is None:
+            lexer = TextLexer
 
         print(lexer.aliases[0])
         return 0
@@ -313,12 +312,7 @@ def main(args=sys.argv):
             print(err, file=sys.stderr)
             return 1
 
-        arg = a_opt or ''
-        try:
-            print(fmter.get_style_defs(arg))
-        except Exception as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
+        print(fmter.get_style_defs(a_opt or ''))
         return 0
 
     # if no -S is given, -a is not allowed
@@ -330,14 +324,35 @@ def main(args=sys.argv):
     F_opts = _parse_filters(F_opts)
     opts.pop('-F', None)
 
+    allow_custom_lexer_formatter = False
+    # -x: allow custom (eXternal) lexers and formatters
+    if opts.pop('-x', None) is not None:
+        allow_custom_lexer_formatter = True
+
     # select lexer
-    lexer = opts.pop('-l', None)
-    if lexer:
-        try:
-            lexer = get_lexer_by_name(lexer, **parsed_opts)
-        except (OptionError, ClassNotFound) as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
+    lexer = None
+
+    # given by name?
+    lexername = opts.pop('-l', None)
+    if lexername:
+        # custom lexer, located relative to user's cwd
+        if allow_custom_lexer_formatter and '.py' in lexername:
+            try:
+                if ':' in lexername:
+                    filename, name = lexername.rsplit(':', 1)
+                    lexer = load_lexer_from_file(filename, name,
+                                                 **parsed_opts)
+                else:
+                    lexer = load_lexer_from_file(lexername, **parsed_opts)
+            except ClassNotFound as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
+        else:
+            try:
+                lexer = get_lexer_by_name(lexername, **parsed_opts)
+            except (OptionError, ClassNotFound) as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
 
     # read input code
     code = None
@@ -350,7 +365,7 @@ def main(args=sys.argv):
         if '-s' in opts:
             print('Error: -s option not usable when input file specified',
                   file=sys.stderr)
-            return 1
+            return 2
 
         infn = args[0]
         try:
@@ -396,15 +411,42 @@ def main(args=sys.argv):
             except ClassNotFound:
                 lexer = TextLexer(**parsed_opts)
 
+    else:  # -s option needs a lexer with -l
+        if not lexer:
+            print('Error: when using -s a lexer has to be selected with -l',
+                  file=sys.stderr)
+            return 2
+
+    # process filters
+    for fname, fopts in F_opts:
+        try:
+            lexer.add_filter(fname, **fopts)
+        except ClassNotFound as err:
+            print('Error:', err, file=sys.stderr)
+            return 1
+
     # select formatter
     outfn = opts.pop('-o', None)
     fmter = opts.pop('-f', None)
     if fmter:
-        try:
-            fmter = get_formatter_by_name(fmter, **parsed_opts)
-        except (OptionError, ClassNotFound) as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
+        # custom formatter, located relative to user's cwd
+        if allow_custom_lexer_formatter and '.py' in fmter:
+            try:
+                if ':' in fmter:
+                    file, fmtername = fmter.rsplit(':', 1)
+                    fmter = load_formatter_from_file(file, fmtername,
+                                                     **parsed_opts)
+                else:
+                    fmter = load_formatter_from_file(fmter, **parsed_opts)
+            except ClassNotFound as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
+        else:
+            try:
+                fmter = get_formatter_by_name(fmter, **parsed_opts)
+            except (OptionError, ClassNotFound) as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
 
     if outfn:
         if not fmter:
@@ -420,7 +462,10 @@ def main(args=sys.argv):
             return 1
     else:
         if not fmter:
-            fmter = TerminalFormatter(**parsed_opts)
+            if '256' in os.environ.get('TERM', ''):
+                fmter = Terminal256Formatter(**parsed_opts)
+            else:
+                fmter = TerminalFormatter(**parsed_opts)
         if sys.version_info > (3,):
             # Python 3: we have to use .buffer to get a binary stream
             outfile = sys.stdout.buffer
@@ -438,11 +483,11 @@ def main(args=sys.argv):
 
     # provide coloring under Windows, if possible
     if not outfn and sys.platform in ('win32', 'cygwin') and \
-       fmter.name in ('Terminal', 'Terminal256'):
+       fmter.name in ('Terminal', 'Terminal256'):  # pragma: no cover
         # unfortunately colorama doesn't support binary streams on Py3
         if sys.version_info > (3,):
-            import io
-            outfile = io.TextIOWrapper(outfile, encoding=fmter.encoding)
+            from pygments.util import UnclosingTextIOWrapper
+            outfile = UnclosingTextIOWrapper(outfile, encoding=fmter.encoding)
             fmter.encoding = None
         try:
             import colorama.initialise
@@ -462,39 +507,58 @@ def main(args=sys.argv):
         lexer = LatexEmbeddedLexer(left, right, lexer)
 
     # ... and do it!
+    if '-s' not in opts:
+        # process whole input as per normal...
+        highlight(code, lexer, fmter, outfile)
+        return 0
+    else:
+        # line by line processing of stdin (eg: for 'tail -f')...
+        try:
+            while 1:
+                if sys.version_info > (3,):
+                    # Python 3: we have to use .buffer to get a binary stream
+                    line = sys.stdin.buffer.readline()
+                else:
+                    line = sys.stdin.readline()
+                if not line:
+                    break
+                if not inencoding:
+                    line = guess_decode_from_terminal(line, sys.stdin)[0]
+                highlight(line, lexer, fmter, outfile)
+                if hasattr(outfile, 'flush'):
+                    outfile.flush()
+            return 0
+        except KeyboardInterrupt:  # pragma: no cover
+            return 0
+
+
+def main(args=sys.argv):
+    """
+    Main command line entry point.
+    """
+    usage = USAGE % ((args[0],) * 6)
+
     try:
-        # process filters
-        for fname, fopts in F_opts:
-            lexer.add_filter(fname, **fopts)
-
-        if '-s' not in opts:
-            # process whole input as per normal...
-            highlight(code, lexer, fmter, outfile)
-        else:
-            if not lexer:
-                print('Error: when using -s a lexer has to be selected with -l',
-                      file=sys.stderr)
-                return 1
-            # line by line processing of stdin (eg: for 'tail -f')...
-            try:
-                while 1:
-                    if sys.version_info > (3,):
-                        # Python 3: we have to use .buffer to get a binary stream
-                        line = sys.stdin.buffer.readline()
-                    else:
-                        line = sys.stdin.readline()
-                    if not line:
-                        break
-                    if not inencoding:
-                        line = guess_decode_from_terminal(line, sys.stdin)[0]
-                    highlight(line, lexer, fmter, outfile)
-                    if hasattr(outfile, 'flush'):
-                        outfile.flush()
-            except KeyboardInterrupt:
-                return 0
+        popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:vhVHgsx")
+    except getopt.GetoptError:
+        print(usage, file=sys.stderr)
+        return 2
 
+    try:
+        return main_inner(popts, args, usage)
     except Exception:
-        raise
+        if '-v' in dict(popts):
+            print(file=sys.stderr)
+            print('*' * 65, file=sys.stderr)
+            print('An unhandled exception occurred while highlighting.',
+                  file=sys.stderr)
+            print('Please report the whole traceback to the issue tracker at',
+                  file=sys.stderr)
+            print('<https://bitbucket.org/birkenfeld/pygments-main/issues>.',
+                  file=sys.stderr)
+            print('*' * 65, file=sys.stderr)
+            print(file=sys.stderr)
+            raise
         import traceback
         info = traceback.format_exception(*sys.exc_info())
         msg = info[-1].strip()
@@ -504,6 +568,6 @@ def main(args=sys.argv):
         print(file=sys.stderr)
         print('*** Error while highlighting:', file=sys.stderr)
         print(msg, file=sys.stderr)
+        print('*** If this is a bug you want to report, please rerun with -v.',
+              file=sys.stderr)
         return 1
-
-    return 0

packages/wakatime/packages/pygments/console.py

@@ -5,25 +5,25 @@
 
     Format colored console output.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 esc = "\x1b["
 
 codes = {}
-codes[""]          = ""
-codes["reset"]     = esc + "39;49;00m"
+codes[""] = ""
+codes["reset"] = esc + "39;49;00m"
 
-codes["bold"]      = esc + "01m"
-codes["faint"]     = esc + "02m"
-codes["standout"]  = esc + "03m"
+codes["bold"] = esc + "01m"
+codes["faint"] = esc + "02m"
+codes["standout"] = esc + "03m"
 codes["underline"] = esc + "04m"
-codes["blink"]     = esc + "05m"
-codes["overline"]  = esc + "06m"
+codes["blink"] = esc + "05m"
+codes["overline"] = esc + "06m"
 
-dark_colors  = ["black", "darkred", "darkgreen", "brown", "darkblue",
-                "purple", "teal", "lightgray"]
+dark_colors = ["black", "darkred", "darkgreen", "brown", "darkblue",
+               "purple", "teal", "lightgray"]
 light_colors = ["darkgray", "red", "green", "yellow", "blue",
                 "fuchsia", "turquoise", "white"]
 
@@ -35,10 +35,10 @@ for d, l in zip(dark_colors, light_colors):
 
 del d, l, x
 
-codes["darkteal"]   = codes["turquoise"]
+codes["darkteal"] = codes["turquoise"]
 codes["darkyellow"] = codes["brown"]
-codes["fuscia"]     = codes["fuchsia"]
-codes["white"]      = codes["bold"]
+codes["fuscia"] = codes["fuchsia"]
+codes["white"] = codes["bold"]
 
 
 def reset_color():

packages/wakatime/packages/pygments/filter.py

@@ -5,7 +5,7 @@
 
     Module that implements the default filter.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -29,15 +29,15 @@ def simplefilter(f):
     Decorator that converts a function into a filter::
 
         @simplefilter
-        def lowercase(lexer, stream, options):
+        def lowercase(self, lexer, stream, options):
             for ttype, value in stream:
                 yield ttype, value.lower()
     """
     return type(f.__name__, (FunctionFilter,), {
-                'function':     f,
-                '__module__':   getattr(f, '__module__'),
-                '__doc__':      f.__doc__
-            })
+        '__module__': getattr(f, '__module__'),
+        '__doc__': f.__doc__,
+        'function': f,
+    })
 
 
 class Filter(object):
@@ -69,6 +69,6 @@ class FunctionFilter(Filter):
         Filter.__init__(self, **options)
 
     def filter(self, lexer, stream):
-        # pylint: disable-msg=E1102
+        # pylint: disable=not-callable
         for ttype, value in self.function(lexer, stream, self.options):
             yield ttype, value

packages/wakatime/packages/pygments/filters/__init__.py

@@ -6,7 +6,7 @@
     Module containing filter lookup functions and default
     filters.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/formatter.py

@@ -5,7 +5,7 @@
 
     Base formatter class.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -65,7 +65,7 @@ class Formatter(object):
 
     def __init__(self, **options):
         self.style = _lookup_style(options.get('style', 'default'))
-        self.full  = get_bool_opt(options, 'full', False)
+        self.full = get_bool_opt(options, 'full', False)
         self.title = options.get('title', '')
         self.encoding = options.get('encoding', None) or None
         if self.encoding in ('guess', 'chardet'):

packages/wakatime/packages/pygments/formatters/__init__.py

@@ -5,7 +5,7 @@
 
     Pygments formatters.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -20,7 +20,7 @@ from pygments.plugin import find_plugin_formatters
 from pygments.util import ClassNotFound, itervalues
 
 __all__ = ['get_formatter_by_name', 'get_formatter_for_filename',
-           'get_all_formatters'] + list(FORMATTERS)
+           'get_all_formatters', 'load_formatter_from_file'] + list(FORMATTERS)
 
 _formatter_cache = {}  # classes by name
 _pattern_cache = {}
@@ -75,10 +75,45 @@ def get_formatter_by_name(_alias, **options):
     """
     cls = find_formatter_class(_alias)
     if cls is None:
-        raise ClassNotFound("No formatter found for name %r" % _alias)
+        raise ClassNotFound("no formatter found for name %r" % _alias)
     return cls(**options)
 
 
+def load_formatter_from_file(filename, formattername="CustomFormatter",
+                             **options):
+    """Load a formatter from a file.
+
+    This method expects a file located relative to the current working
+    directory, which contains a class named CustomFormatter. By default,
+    it expects the Formatter to be named CustomFormatter; you can specify
+    your own class name as the second argument to this function.
+
+    Users should be very careful with the input, because this method
+    is equivalent to running eval on the input file.
+
+    Raises ClassNotFound if there are any problems importing the Formatter.
+
+    .. versionadded:: 2.2
+    """
+    try:
+        # This empty dict will contain the namespace for the exec'd file
+        custom_namespace = {}
+        exec(open(filename, 'rb').read(), custom_namespace)
+        # Retrieve the class `formattername` from that namespace
+        if formattername not in custom_namespace:
+            raise ClassNotFound('no valid %s class found in %s' %
+                                (formattername, filename))
+        formatter_class = custom_namespace[formattername]
+        # And finally instantiate it with the options
+        return formatter_class(**options)
+    except IOError as err:
+        raise ClassNotFound('cannot read %s' % filename)
+    except ClassNotFound as err:
+        raise
+    except Exception as err:
+        raise ClassNotFound('error when loading custom formatter: %s' % err)
+
+
 def get_formatter_for_filename(fn, **options):
     """Lookup and instantiate a formatter by filename pattern.
 
@@ -95,7 +130,7 @@ def get_formatter_for_filename(fn, **options):
         for filename in cls.filenames:
             if _fn_matches(fn, filename):
                 return cls(**options)
-    raise ClassNotFound("No formatter found for file name %r" % fn)
+    raise ClassNotFound("no formatter found for file name %r" % fn)
 
 
 class _automodule(types.ModuleType):

packages/wakatime/packages/pygments/formatters/_mapping.py

@@ -9,7 +9,7 @@
 
     Do not alter the FORMATTERS dictionary by hand.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -20,6 +20,7 @@ FORMATTERS = {
     'BmpImageFormatter': ('pygments.formatters.img', 'img_bmp', ('bmp', 'bitmap'), ('*.bmp',), 'Create a bitmap image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
     'GifImageFormatter': ('pygments.formatters.img', 'img_gif', ('gif',), ('*.gif',), 'Create a GIF image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
     'HtmlFormatter': ('pygments.formatters.html', 'HTML', ('html',), ('*.html', '*.htm'), "Format tokens as HTML 4 ``<span>`` tags within a ``<pre>`` tag, wrapped in a ``<div>`` tag. The ``<div>``'s CSS class can be set by the `cssclass` option."),
+    'IRCFormatter': ('pygments.formatters.irc', 'IRC', ('irc', 'IRC'), (), 'Format tokens with IRC color sequences'),
     'ImageFormatter': ('pygments.formatters.img', 'img', ('img', 'IMG', 'png'), ('*.png',), 'Create a PNG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
     'JpgImageFormatter': ('pygments.formatters.img', 'img_jpg', ('jpg', 'jpeg'), ('*.jpg',), 'Create a JPEG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
     'LatexFormatter': ('pygments.formatters.latex', 'LaTeX', ('latex', 'tex'), ('*.tex',), 'Format tokens as LaTeX code. This needs the `fancyvrb` and `color` standard packages.'),
@@ -27,12 +28,13 @@ FORMATTERS = {
     'RawTokenFormatter': ('pygments.formatters.other', 'Raw tokens', ('raw', 'tokens'), ('*.raw',), 'Format tokens as a raw representation for storing token streams.'),
     'RtfFormatter': ('pygments.formatters.rtf', 'RTF', ('rtf',), ('*.rtf',), 'Format tokens as RTF markup. This formatter automatically outputs full RTF documents with color information and other useful stuff. Perfect for Copy and Paste into Microsoft(R) Word(R) documents.'),
     'SvgFormatter': ('pygments.formatters.svg', 'SVG', ('svg',), ('*.svg',), 'Format tokens as an SVG graphics file.  This formatter is still experimental. Each line of code is a ``<text>`` element with explicit ``x`` and ``y`` coordinates containing ``<tspan>`` elements with the individual token styles.'),
-    'Terminal256Formatter': ('pygments.formatters.terminal256', 'Terminal256', ('terminal256', 'console256', '256'), (), 'Format tokens with ANSI color sequences, for output in a 256-color terminal or console. Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'),
+    'Terminal256Formatter': ('pygments.formatters.terminal256', 'Terminal256', ('terminal256', 'console256', '256'), (), 'Format tokens with ANSI color sequences, for output in a 256-color terminal or console.  Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'),
     'TerminalFormatter': ('pygments.formatters.terminal', 'Terminal', ('terminal', 'console'), (), 'Format tokens with ANSI color sequences, for output in a text console. Color sequences are terminated at newlines, so that paging the output works correctly.'),
+    'TerminalTrueColorFormatter': ('pygments.formatters.terminal256', 'TerminalTrueColor', ('terminal16m', 'console16m', '16m'), (), 'Format tokens with ANSI color sequences, for output in a true-color terminal or console.  Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'),
     'TestcaseFormatter': ('pygments.formatters.other', 'Testcase', ('testcase',), (), 'Format tokens as appropriate for a new testcase.')
 }
 
-if __name__ == '__main__':
+if __name__ == '__main__':  # pragma: no cover
     import sys
     import os
 
@@ -64,6 +66,13 @@ if __name__ == '__main__':
     # extract useful sourcecode from this file
     with open(__file__) as fp:
         content = fp.read()
+        # replace crnl to nl for Windows.
+        #
+        # Note that, originally, contributers should keep nl of master
+        # repository, for example by using some kind of automatic
+        # management EOL, like `EolExtension
+        #  <https://www.mercurial-scm.org/wiki/EolExtension>`.
+        content = content.replace("\r\n", "\n")
     header = content[:content.find('FORMATTERS = {')]
     footer = content[content.find("if __name__ == '__main__':"):]
 

packages/wakatime/packages/pygments/formatters/bbcode.py

@@ -5,7 +5,7 @@
 
     BBcode formatter.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/formatters/html.py

@@ -5,7 +5,7 @@
 
     Formatter for HTML output.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -36,21 +36,11 @@ _escape_html_table = {
     ord("'"): u''',
 }
 
+
 def escape_html(text, table=_escape_html_table):
     """Escape &, <, > as well as single and double quotes for HTML."""
     return text.translate(table)
 
-def get_random_id():
-    """Return a random id for javascript fields."""
-    from random import random
-    from time import time
-    try:
-        from hashlib import sha1 as sha
-    except ImportError:
-        import sha
-        sha = sha.new
-    return sha('%s|%s' % (random(), time())).hexdigest()
-
 
 def _get_ttype_class(ttype):
     fname = STANDARD_TYPES.get(ttype)
@@ -150,7 +140,7 @@ class HtmlFormatter(Formatter):
 
     When `tagsfile` is set to the path of a ctags index file, it is used to
     generate hyperlinks from names to their definition.  You must enable
-    `anchorlines` and run ctags with the `-n` option for this to work.  The
+    `lineanchors` and run ctags with the `-n` option for this to work.  The
     `python-ctags` module from PyPI must be installed to use this feature;
     otherwise a `RuntimeError` will be raised.
 
@@ -331,6 +321,12 @@ class HtmlFormatter(Formatter):
 
         .. versionadded:: 1.6
 
+    `filename`
+        A string used to generate a filename when rendering <pre> blocks,
+        for example if displaying source code.
+
+        .. versionadded:: 2.1
+
 
     **Subclassing the HTML formatter**
 
@@ -398,6 +394,7 @@ class HtmlFormatter(Formatter):
         self.noclobber_cssfile = get_bool_opt(options, 'noclobber_cssfile', False)
         self.tagsfile = self._decodeifneeded(options.get('tagsfile', ''))
         self.tagurlformat = self._decodeifneeded(options.get('tagurlformat', ''))
+        self.filename = self._decodeifneeded(options.get('filename', ''))
 
         if self.tagsfile:
             if not ctags:
@@ -438,6 +435,15 @@ class HtmlFormatter(Formatter):
             return self.classprefix + ttypeclass
         return ''
 
+    def _get_css_classes(self, ttype):
+        """Return the css classes of this token type prefixed with
+        the classprefix option."""
+        cls = self._get_css_class(ttype)
+        while ttype not in STANDARD_TYPES:
+            ttype = ttype.parent
+            cls = self._get_css_class(ttype) + ' ' + cls
+        return cls
+
     def _create_stylesheet(self):
         t2c = self.ttype2class = {Token: ''}
         c2s = self.class2style = {}
@@ -522,7 +528,7 @@ class HtmlFormatter(Formatter):
                     cssfilename = os.path.join(os.path.dirname(filename),
                                                self.cssfile)
                 except AttributeError:
-                    print('Note: Cannot determine output file name, ' \
+                    print('Note: Cannot determine output file name, '
                           'using current directory as base for the CSS file name',
                           file=sys.stderr)
                     cssfilename = self.cssfile
@@ -531,21 +537,21 @@ class HtmlFormatter(Formatter):
                 if not os.path.exists(cssfilename) or not self.noclobber_cssfile:
                     cf = open(cssfilename, "w")
                     cf.write(CSSFILE_TEMPLATE %
-                            {'styledefs': self.get_style_defs('body')})
+                             {'styledefs': self.get_style_defs('body')})
                     cf.close()
             except IOError as err:
                 err.strerror = 'Error writing CSS file: ' + err.strerror
                 raise
 
             yield 0, (DOC_HEADER_EXTERNALCSS %
-                      dict(title     = self.title,
-                           cssfile   = self.cssfile,
-                           encoding  = self.encoding))
+                      dict(title=self.title,
+                           cssfile=self.cssfile,
+                           encoding=self.encoding))
         else:
             yield 0, (DOC_HEADER %
-                      dict(title     = self.title,
-                           styledefs = self.get_style_defs('body'),
-                           encoding  = self.encoding))
+                      dict(title=self.title,
+                           styledefs=self.get_style_defs('body'),
+                           encoding=self.encoding))
 
         for t, line in inner:
             yield t, line
@@ -624,35 +630,35 @@ class HtmlFormatter(Formatter):
         if self.noclasses:
             if sp:
                 for t, line in lines:
-                    if num%sp == 0:
+                    if num % sp == 0:
                         style = 'background-color: #ffffc0; padding: 0 5px 0 5px'
                     else:
                         style = 'background-color: #f0f0f0; padding: 0 5px 0 5px'
                     yield 1, '<span style="%s">%*s </span>' % (
-                        style, mw, (num%st and ' ' or num)) + line
+                        style, mw, (num % st and ' ' or num)) + line
                     num += 1
             else:
                 for t, line in lines:
                     yield 1, ('<span style="background-color: #f0f0f0; '
                               'padding: 0 5px 0 5px">%*s </span>' % (
-                              mw, (num%st and ' ' or num)) + line)
+                                  mw, (num % st and ' ' or num)) + line)
                     num += 1
         elif sp:
             for t, line in lines:
                 yield 1, '<span class="lineno%s">%*s </span>' % (
-                    num%sp == 0 and ' special' or '', mw,
-                    (num%st and ' ' or num)) + line
+                    num % sp == 0 and ' special' or '', mw,
+                    (num % st and ' ' or num)) + line
                 num += 1
         else:
             for t, line in lines:
                 yield 1, '<span class="lineno">%*s </span>' % (
-                    mw, (num%st and ' ' or num)) + line
+                    mw, (num % st and ' ' or num)) + line
                 num += 1
 
     def _wrap_lineanchors(self, inner):
         s = self.lineanchors
-        i = self.linenostart - 1 # subtract 1 since we have to increment i
-                                 # *before* yielding
+        # subtract 1 since we have to increment i *before* yielding
+        i = self.linenostart - 1
         for t, line in inner:
             if t:
                 i += 1
@@ -673,14 +679,14 @@ class HtmlFormatter(Formatter):
     def _wrap_div(self, inner):
         style = []
         if (self.noclasses and not self.nobackground and
-            self.style.background_color is not None):
+                self.style.background_color is not None):
             style.append('background: %s' % (self.style.background_color,))
         if self.cssstyles:
             style.append(self.cssstyles)
         style = '; '.join(style)
 
-        yield 0, ('<div' + (self.cssclass and ' class="%s"' % self.cssclass)
-                  + (style and (' style="%s"' % style)) + '>')
+        yield 0, ('<div' + (self.cssclass and ' class="%s"' % self.cssclass) +
+                  (style and (' style="%s"' % style)) + '>')
         for tup in inner:
             yield tup
         yield 0, '</div>\n'
@@ -693,7 +699,12 @@ class HtmlFormatter(Formatter):
             style.append('line-height: 125%')
         style = '; '.join(style)
 
-        yield 0, ('<pre' + (style and ' style="%s"' % style) + '>')
+        if self.filename:
+            yield 0, ('<span class="filename">' + self.filename + '</span>')
+
+        # the empty span here is to keep leading empty lines from being
+        # ignored by HTML parsers
+        yield 0, ('<pre' + (style and ' style="%s"' % style) + '><span></span>')
         for tup in inner:
             yield tup
         yield 0, '</pre>'
@@ -712,7 +723,7 @@ class HtmlFormatter(Formatter):
         tagsfile = self.tagsfile
 
         lspan = ''
-        line = ''
+        line = []
         for ttype, value in tokensource:
             if nocls:
                 cclass = getcls(ttype)
@@ -721,7 +732,7 @@ class HtmlFormatter(Formatter):
                     cclass = getcls(ttype)
                 cspan = cclass and '<span style="%s">' % c2s[cclass][0] or ''
             else:
-                cls = self._get_css_class(ttype)
+                cls = self._get_css_classes(ttype)
                 cspan = cls and '<span class="%s">' % cls or ''
 
             parts = value.translate(escape_table).split('\n')
@@ -743,30 +754,31 @@ class HtmlFormatter(Formatter):
             for part in parts[:-1]:
                 if line:
                     if lspan != cspan:
-                        line += (lspan and '</span>') + cspan + part + \
-                                (cspan and '</span>') + lsep
-                    else: # both are the same
-                        line += part + (lspan and '</span>') + lsep
-                    yield 1, line
-                    line = ''
+                        line.extend(((lspan and '</span>'), cspan, part,
+                                     (cspan and '</span>'), lsep))
+                    else:  # both are the same
+                        line.extend((part, (lspan and '</span>'), lsep))
+                    yield 1, ''.join(line)
+                    line = []
                 elif part:
-                    yield 1, cspan + part + (cspan and '</span>') + lsep
+                    yield 1, ''.join((cspan, part, (cspan and '</span>'), lsep))
                 else:
                     yield 1, lsep
             # for the last line
             if line and parts[-1]:
                 if lspan != cspan:
-                    line += (lspan and '</span>') + cspan + parts[-1]
+                    line.extend(((lspan and '</span>'), cspan, parts[-1]))
                     lspan = cspan
                 else:
-                    line += parts[-1]
+                    line.append(parts[-1])
             elif parts[-1]:
-                line = cspan + parts[-1]
+                line = [cspan, parts[-1]]
                 lspan = cspan
             # else we neither have to open a new span nor set lspan
 
         if line:
-            yield 1, line + (lspan and '</span>') + lsep
+            line.extend(((lspan and '</span>'), lsep))
+            yield 1, ''.join(line)
 
     def _lookup_ctag(self, token):
         entry = ctags.TagEntry()
@@ -785,7 +797,7 @@ class HtmlFormatter(Formatter):
         for i, (t, value) in enumerate(tokensource):
             if t != 1:
                 yield t, value
-            if i + 1 in hls: # i + 1 because Python indexes start at 0
+            if i + 1 in hls:  # i + 1 because Python indexes start at 0
                 if self.noclasses:
                     style = ''
                     if self.style.highlight_color is not None:

packages/wakatime/packages/pygments/formatters/img.py

@@ -5,16 +5,19 @@
 
     Formatter for Pixmap output.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+import os
 import sys
 
 from pygments.formatter import Formatter
 from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
     get_choice_opt, xrange
 
+import subprocess
+
 # Import this carefully
 try:
     from PIL import Image, ImageDraw, ImageFont
@@ -45,6 +48,7 @@ STYLES = {
 # A sane default for modern systems
 DEFAULT_FONT_NAME_NIX = 'Bitstream Vera Sans Mono'
 DEFAULT_FONT_NAME_WIN = 'Courier New'
+DEFAULT_FONT_NAME_MAC = 'Courier New'
 
 
 class PilNotAvailable(ImportError):
@@ -69,23 +73,28 @@ class FontManager(object):
             if not font_name:
                 self.font_name = DEFAULT_FONT_NAME_WIN
             self._create_win()
+        elif sys.platform.startswith('darwin'):
+            if not font_name:
+                self.font_name = DEFAULT_FONT_NAME_MAC
+            self._create_mac()
         else:
             if not font_name:
                 self.font_name = DEFAULT_FONT_NAME_NIX
             self._create_nix()
 
     def _get_nix_font_path(self, name, style):
-        try:
-            from commands import getstatusoutput
-        except ImportError:
-            from subprocess import getstatusoutput
-        exit, out = getstatusoutput('fc-list "%s:style=%s" file' %
-                                    (name, style))
-        if not exit:
-            lines = out.splitlines()
-            if lines:
-                path = lines[0].strip().strip(':')
-                return path
+        proc = subprocess.Popen(['fc-list', "%s:style=%s" % (name, style), 'file'],
+                                stdout=subprocess.PIPE, stderr=None)
+        stdout, _ = proc.communicate()
+        if proc.returncode == 0:
+            lines = stdout.splitlines()
+            for line in lines:
+                if line.startswith(b'Fontconfig warning:'):
+                    continue
+                path = line.decode().strip().strip(':')
+                if path:
+                    return path
+            return None
 
     def _create_nix(self):
         for name in STYLES['NORMAL']:
@@ -108,6 +117,37 @@ class FontManager(object):
                 else:
                     self.fonts[style] = self.fonts['NORMAL']
 
+    def _get_mac_font_path(self, font_map, name, style):
+        return font_map.get((name + ' ' + style).strip().lower())
+
+    def _create_mac(self):
+        font_map = {}
+        for font_dir in (os.path.join(os.getenv("HOME"), 'Library/Fonts/'),
+                         '/Library/Fonts/', '/System/Library/Fonts/'):
+            font_map.update(
+                ((os.path.splitext(f)[0].lower(), os.path.join(font_dir, f))
+                    for f in os.listdir(font_dir) if f.lower().endswith('ttf')))
+
+        for name in STYLES['NORMAL']:
+            path = self._get_mac_font_path(font_map, self.font_name, name)
+            if path is not None:
+                self.fonts['NORMAL'] = ImageFont.truetype(path, self.font_size)
+                break
+        else:
+            raise FontNotFound('No usable fonts named: "%s"' %
+                               self.font_name)
+        for style in ('ITALIC', 'BOLD', 'BOLDITALIC'):
+            for stylename in STYLES[style]:
+                path = self._get_mac_font_path(font_map, self.font_name, stylename)
+                if path is not None:
+                    self.fonts[style] = ImageFont.truetype(path, self.font_size)
+                    break
+            else:
+                if style == 'BOLDITALIC':
+                    self.fonts[style] = self.fonts['BOLD']
+                else:
+                    self.fonts[style] = self.fonts['NORMAL']
+
     def _lookup_win(self, key, basename, styles, fail=False):
         for suffix in ('', ' (TrueType)'):
             for style in styles:
@@ -197,7 +237,7 @@ class ImageFormatter(Formatter):
         bold and italic fonts will be generated.  This really should be a
         monospace font to look sane.
 
-        Default: "Bitstream Vera Sans Mono"
+        Default: "Bitstream Vera Sans Mono" on Windows, Courier New on \*nix
 
     `font_size`
         The font size in points to be used.

packages/wakatime/packages/pygments/formatters/irc.py

@@ -1,11 +1,11 @@
 # -*- coding: utf-8 -*-
 """
-    pygments.formatters.terminal
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    pygments.formatters.irc
+    ~~~~~~~~~~~~~~~~~~~~~~~
 
-    Formatter for terminal output with ANSI sequences.
+    Formatter for IRC output
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -14,16 +14,15 @@ import sys
 from pygments.formatter import Formatter
 from pygments.token import Keyword, Name, Comment, String, Error, \
     Number, Operator, Generic, Token, Whitespace
-from pygments.console import ansiformat
 from pygments.util import get_choice_opt
 
 
-__all__ = ['TerminalFormatter']
+__all__ = ['IRCFormatter']
 
 
 #: Map token types to a tuple of color values for light and dark
 #: backgrounds.
-TERMINAL_COLORS = {
+IRC_COLORS = {
     Token:              ('',            ''),
 
     Whitespace:         ('lightgray',   'darkgray'),
@@ -55,11 +54,52 @@ TERMINAL_COLORS = {
 }
 
 
-class TerminalFormatter(Formatter):
+IRC_COLOR_MAP = {
+    'white': 0,
+    'black': 1,
+    'darkblue': 2,
+    'green': 3,
+    'red': 4,
+    'brown': 5,
+    'purple': 6,
+    'orange': 7,
+    'darkgreen': 7, #compat w/ ansi
+    'yellow': 8,
+    'lightgreen': 9,
+    'turquoise': 9, # compat w/ ansi
+    'teal': 10,
+    'lightblue': 11,
+    'darkred': 11, # compat w/ ansi
+    'blue': 12,
+    'fuchsia': 13,
+    'darkgray': 14,
+    'lightgray': 15,
+}
+
+def ircformat(color, text):
+    if len(color) < 1:
+        return text
+    add = sub = ''
+    if '_' in color: # italic
+        add += '\x1D'
+        sub = '\x1D' + sub
+        color = color.strip('_')
+    if '*' in color: # bold
+        add += '\x02'
+        sub = '\x02' + sub
+        color = color.strip('*')
+    # underline (\x1F) not supported
+    # backgrounds (\x03FF,BB) not supported
+    if len(color) > 0: # actual color - may have issues with ircformat("red", "blah")+"10" type stuff
+        add += '\x03' + str(IRC_COLOR_MAP[color]).zfill(2)
+        sub = '\x03' + sub
+    return add + text + sub
+    return '<'+add+'>'+text+'</'+sub+'>'
+
+
+class IRCFormatter(Formatter):
     r"""
-    Format tokens with ANSI color sequences, for output in a text console.
-    Color sequences are terminated at newlines, so that paging the output
-    works correctly.
+    Format tokens with IRC color sequences
 
     The `get_style_defs()` method doesn't do anything special since there is
     no support for common styles.
@@ -75,30 +115,21 @@ class TerminalFormatter(Formatter):
         ``None`` (default: ``None`` = use builtin colorscheme).
 
     `linenos`
-        Set to ``True`` to have line numbers on the terminal output as well
+        Set to ``True`` to have line numbers in the output as well
         (default: ``False`` = no line numbers).
     """
-    name = 'Terminal'
-    aliases = ['terminal', 'console']
+    name = 'IRC'
+    aliases = ['irc', 'IRC']
     filenames = []
 
     def __init__(self, **options):
         Formatter.__init__(self, **options)
         self.darkbg = get_choice_opt(options, 'bg',
                                      ['light', 'dark'], 'light') == 'dark'
-        self.colorscheme = options.get('colorscheme', None) or TERMINAL_COLORS
+        self.colorscheme = options.get('colorscheme', None) or IRC_COLORS
         self.linenos = options.get('linenos', False)
         self._lineno = 0
 
-    def format(self, tokensource, outfile):
-        # hack: if the output is a terminal and has an encoding set,
-        # use that to avoid unicode encode problems
-        if not self.encoding and hasattr(outfile, "encoding") and \
-           hasattr(outfile, "isatty") and outfile.isatty() and \
-           sys.version_info < (3,):
-            self.encoding = outfile.encoding
-        return Formatter.format(self, tokensource, outfile)
-
     def _write_lineno(self, outfile):
         self._lineno += 1
         outfile.write("\n%04d: " % self._lineno)
@@ -120,9 +151,9 @@ class TerminalFormatter(Formatter):
                 for line in spl[:-1]:
                     self._write_lineno(outfile)
                     if line:
-                        outfile.write(ansiformat(color, line[:-1]))
+                        outfile.write(ircformat(color, line[:-1]))
                 if spl[-1]:
-                    outfile.write(ansiformat(color, spl[-1]))
+                    outfile.write(ircformat(color, spl[-1]))
             else:
                 outfile.write(value)
 
@@ -143,9 +174,9 @@ class TerminalFormatter(Formatter):
                 spl = value.split('\n')
                 for line in spl[:-1]:
                     if line:
-                        outfile.write(ansiformat(color, line))
+                        outfile.write(ircformat(color, line))
                     outfile.write('\n')
                 if spl[-1]:
-                    outfile.write(ansiformat(color, spl[-1]))
+                    outfile.write(ircformat(color, spl[-1]))
             else:
                 outfile.write(value)

packages/wakatime/packages/pygments/formatters/latex.py

@@ -5,7 +5,7 @@
 
     Formatter for LaTeX fancyvrb output.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -360,7 +360,7 @@ class LatexFormatter(Formatter):
                         start += value[i]
 
                     value = value[len(start):]
-                    start = escape_tex(start, self.commandprefix)
+                    start = escape_tex(start, cp)
 
                     # ... but do not escape inside comment.
                     value = start + value
@@ -370,26 +370,26 @@ class LatexFormatter(Formatter):
                     in_math = False
                     for i, part in enumerate(parts):
                         if not in_math:
-                            parts[i] = escape_tex(part, self.commandprefix)
+                            parts[i] = escape_tex(part, cp)
                         in_math = not in_math
                     value = '$'.join(parts)
                 elif self.escapeinside:
                     text = value
                     value = ''
-                    while len(text) > 0:
+                    while text:
                         a, sep1, text = text.partition(self.left)
-                        if len(sep1) > 0:
+                        if sep1:
                             b, sep2, text = text.partition(self.right)
-                            if len(sep2) > 0:
-                                value += escape_tex(a, self.commandprefix) + b
+                            if sep2:
+                                value += escape_tex(a, cp) + b
                             else:
-                                value += escape_tex(a + sep1 + b, self.commandprefix)
+                                value += escape_tex(a + sep1 + b, cp)
                         else:
-                            value = value + escape_tex(a, self.commandprefix)
+                            value += escape_tex(a, cp)
                 else:
-                    value = escape_tex(value, self.commandprefix)
+                    value = escape_tex(value, cp)
             elif ttype not in Token.Escape:
-                value = escape_tex(value, self.commandprefix)
+                value = escape_tex(value, cp)
             styles = []
             while ttype is not Token:
                 try:
@@ -413,18 +413,24 @@ class LatexFormatter(Formatter):
         outfile.write(u'\\end{' + self.envname + u'}\n')
 
         if self.full:
+            encoding = self.encoding or 'utf8'
+            # map known existings encodings from LaTeX distribution
+            encoding = {
+                'utf_8': 'utf8',
+                'latin_1': 'latin1',
+                'iso_8859_1': 'latin1',
+            }.get(encoding.replace('-', '_'), encoding)
             realoutfile.write(DOC_TEMPLATE %
                 dict(docclass  = self.docclass,
                      preamble  = self.preamble,
                      title     = self.title,
-                     encoding  = self.encoding or 'utf8',
+                     encoding  = encoding,
                      styledefs = self.get_style_defs(),
                      code      = outfile.getvalue()))
 
 
 class LatexEmbeddedLexer(Lexer):
-    r"""
-
+    """
     This lexer takes one lexer as argument, the lexer for the language
     being formatted, and the left and right delimiters for escaped text.
 

packages/wakatime/packages/pygments/formatters/other.py

@@ -5,7 +5,7 @@
 
     Other formatters: NullFormatter, RawTokenFormatter.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/formatters/rtf.py

@@ -5,7 +5,7 @@
 
     A formatter that generates RTF files.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/formatters/svg.py

@@ -5,7 +5,7 @@
 
     Formatter for SVG output.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/formatters/terminal.py

@@ -5,7 +5,7 @@
 
     Formatter for terminal output with ANSI sequences.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -49,6 +49,7 @@ TERMINAL_COLORS = {
     Generic.Inserted:   ('darkgreen',  'green'),
     Generic.Heading:    ('**',         '**'),
     Generic.Subheading: ('*purple*',   '*fuchsia*'),
+    Generic.Prompt:     ('**',         '**'),
     Generic.Error:      ('red',        'red'),
 
     Error:              ('_red_',      '_red_'),
@@ -101,51 +102,35 @@ class TerminalFormatter(Formatter):
 
     def _write_lineno(self, outfile):
         self._lineno += 1
-        outfile.write("\n%04d: " % self._lineno)
+        outfile.write("%s%04d: " % (self._lineno != 1 and '\n' or '', self._lineno))
 
-    def _format_unencoded_with_lineno(self, tokensource, outfile):
-        self._write_lineno(outfile)
-
-        for ttype, value in tokensource:
-            if value.endswith("\n"):
-                self._write_lineno(outfile)
-                value = value[:-1]
-            color = self.colorscheme.get(ttype)
-            while color is None:
-                ttype = ttype[:-1]
-                color = self.colorscheme.get(ttype)
-            if color:
-                color = color[self.darkbg]
-                spl = value.split('\n')
-                for line in spl[:-1]:
-                    self._write_lineno(outfile)
-                    if line:
-                        outfile.write(ansiformat(color, line[:-1]))
-                if spl[-1]:
-                    outfile.write(ansiformat(color, spl[-1]))
-            else:
-                outfile.write(value)
-
-        outfile.write("\n")
+    def _get_color(self, ttype):
+        # self.colorscheme is a dict containing usually generic types, so we
+        # have to walk the tree of dots.  The base Token type must be a key,
+        # even if it's empty string, as in the default above.
+        colors = self.colorscheme.get(ttype)
+        while colors is None:
+            ttype = ttype.parent
+            colors = self.colorscheme.get(ttype)
+        return colors[self.darkbg]
 
     def format_unencoded(self, tokensource, outfile):
         if self.linenos:
-            self._format_unencoded_with_lineno(tokensource, outfile)
-            return
+            self._write_lineno(outfile)
 
         for ttype, value in tokensource:
-            color = self.colorscheme.get(ttype)
-            while color is None:
-                ttype = ttype[:-1]
-                color = self.colorscheme.get(ttype)
-            if color:
-                color = color[self.darkbg]
-                spl = value.split('\n')
-                for line in spl[:-1]:
-                    if line:
-                        outfile.write(ansiformat(color, line))
-                    outfile.write('\n')
-                if spl[-1]:
-                    outfile.write(ansiformat(color, spl[-1]))
-            else:
-                outfile.write(value)
+            color = self._get_color(ttype)
+
+            for line in value.splitlines(True):
+                if color:
+                    outfile.write(ansiformat(color, line.rstrip('\n')))
+                else:
+                    outfile.write(line.rstrip('\n'))
+                if line.endswith('\n'):
+                    if self.linenos:
+                        self._write_lineno(outfile)
+                    else:
+                        outfile.write('\n')
+
+        if self.linenos:
+            outfile.write("\n")

packages/wakatime/packages/pygments/formatters/terminal256.py

@@ -11,7 +11,7 @@
 
     Formatter version 1.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -27,9 +27,11 @@
 import sys
 
 from pygments.formatter import Formatter
+from pygments.console import codes
+from pygments.style import ansicolors
 
 
-__all__ = ['Terminal256Formatter']
+__all__ = ['Terminal256Formatter', 'TerminalTrueColorFormatter']
 
 
 class EscapeSequence:
@@ -47,9 +49,33 @@ class EscapeSequence:
     def color_string(self):
         attrs = []
         if self.fg is not None:
-            attrs.extend(("38", "5", "%i" % self.fg))
+            if self.fg in ansicolors:
+                esc = codes[self.fg[5:]]
+                if ';01m' in esc:
+                    self.bold = True
+                # extract fg color code.
+                attrs.append(esc[2:4])
+            else:
+                attrs.extend(("38", "5", "%i" % self.fg))
         if self.bg is not None:
-            attrs.extend(("48", "5", "%i" % self.bg))
+            if self.bg in ansicolors:
+                esc = codes[self.bg[5:]]
+                # extract fg color code, add 10 for bg.
+                attrs.append(str(int(esc[2:4])+10))
+            else:
+                attrs.extend(("48", "5", "%i" % self.bg))
+        if self.bold:
+            attrs.append("01")
+        if self.underline:
+            attrs.append("04")
+        return self.escape(attrs)
+
+    def true_color_string(self):
+        attrs = []
+        if self.fg:
+            attrs.extend(("38", "2", str(self.fg[0]), str(self.fg[1]), str(self.fg[2])))
+        if self.bg:
+            attrs.extend(("48", "2", str(self.bg[0]), str(self.bg[1]), str(self.bg[2])))
         if self.bold:
             attrs.append("01")
         if self.underline:
@@ -68,9 +94,9 @@ class EscapeSequence:
 
 
 class Terminal256Formatter(Formatter):
-    r"""
+    """
     Format tokens with ANSI color sequences, for output in a 256-color
-    terminal or console. Like in `TerminalFormatter` color sequences
+    terminal or console.  Like in `TerminalFormatter` color sequences
     are terminated at newlines, so that paging the output works correctly.
 
     The formatter takes colors from a style defined by the `style` option
@@ -79,6 +105,11 @@ class Terminal256Formatter(Formatter):
 
     .. versionadded:: 0.9
 
+    .. versionchanged:: 2.2
+       If the used style defines foreground colors in the form ``#ansi*``, then
+       `Terminal256Formatter` will map these to non extended foreground color.
+       See :ref:`AnsiTerminalStyle` for more information.
+
     Options accepted:
 
     `style`
@@ -157,6 +188,10 @@ class Terminal256Formatter(Formatter):
 
     def _color_index(self, color):
         index = self.best_match.get(color, None)
+        if color in ansicolors:
+            # strip the `#ansi` part and look up code
+            index = color
+            self.best_match[color] = index
         if index is None:
             try:
                 rgb = int(str(color), 16)
@@ -173,9 +208,14 @@ class Terminal256Formatter(Formatter):
     def _setup_styles(self):
         for ttype, ndef in self.style:
             escape = EscapeSequence()
-            if ndef['color']:
+            # get foreground from ansicolor if set
+            if ndef['ansicolor']:
+                escape.fg = self._color_index(ndef['ansicolor'])
+            elif ndef['color']:
                 escape.fg = self._color_index(ndef['color'])
-            if ndef['bgcolor']:
+            if ndef['bgansicolor']:
+                escape.bg = self._color_index(ndef['bgansicolor'])
+            elif ndef['bgcolor']:
                 escape.bg = self._color_index(ndef['bgcolor'])
             if self.usebold and ndef['bold']:
                 escape.bold = True
@@ -221,3 +261,49 @@ class Terminal256Formatter(Formatter):
 
             if not_found:
                 outfile.write(value)
+
+
+class TerminalTrueColorFormatter(Terminal256Formatter):
+    r"""
+    Format tokens with ANSI color sequences, for output in a true-color
+    terminal or console.  Like in `TerminalFormatter` color sequences
+    are terminated at newlines, so that paging the output works correctly.
+
+    .. versionadded:: 2.1
+
+    Options accepted:
+
+    `style`
+        The style to use, can be a string or a Style subclass (default:
+        ``'default'``).
+    """
+    name = 'TerminalTrueColor'
+    aliases = ['terminal16m', 'console16m', '16m']
+    filenames = []
+
+    def _build_color_table(self):
+        pass
+
+    def _color_tuple(self, color):
+        try:
+            rgb = int(str(color), 16)
+        except ValueError:
+            return None
+        r = (rgb >> 16) & 0xff
+        g = (rgb >> 8) & 0xff
+        b = rgb & 0xff
+        return (r, g, b)
+
+    def _setup_styles(self):
+        for ttype, ndef in self.style:
+            escape = EscapeSequence()
+            if ndef['color']:
+                escape.fg = self._color_tuple(ndef['color'])
+            if ndef['bgcolor']:
+                escape.bg = self._color_tuple(ndef['bgcolor'])
+            if self.usebold and ndef['bold']:
+                escape.bold = True
+            if self.useunderline and ndef['underline']:
+                escape.underline = True
+            self.style_string[str(ttype)] = (escape.true_color_string(),
+                                             escape.reset_string())

packages/wakatime/packages/pygments/lexer.py

@@ -5,7 +5,7 @@
 
     Base lexer classes.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -14,7 +14,6 @@ from __future__ import print_function
 import re
 import sys
 import time
-import itertools
 
 from pygments.filter import apply_filters, Filter
 from pygments.filters import get_filter_by_name
@@ -43,10 +42,10 @@ class LexerMeta(type):
     static methods which always return float values.
     """
 
-    def __new__(cls, name, bases, d):
+    def __new__(mcs, name, bases, d):
         if 'analyse_text' in d:
             d['analyse_text'] = make_analysator(d['analyse_text'])
-        return type.__new__(cls, name, bases, d)
+        return type.__new__(mcs, name, bases, d)
 
 
 @add_metaclass(LexerMeta)
@@ -189,7 +188,7 @@ class Lexer(object):
             text += '\n'
 
         def streamer():
-            for i, t, v in self.get_tokens_unprocessed(text):
+            for _, t, v in self.get_tokens_unprocessed(text):
                 yield t, v
         stream = streamer()
         if not unfiltered:
@@ -246,7 +245,7 @@ class DelegatingLexer(Lexer):
 #
 
 
-class include(str):
+class include(str):  # pylint: disable=invalid-name
     """
     Indicates that a state should include rules from another state.
     """
@@ -260,10 +259,10 @@ class _inherit(object):
     def __repr__(self):
         return 'inherit'
 
-inherit = _inherit()
+inherit = _inherit()  # pylint: disable=invalid-name
 
 
-class combined(tuple):
+class combined(tuple):  # pylint: disable=invalid-name
     """
     Indicates a state combined from multiple states.
     """
@@ -320,8 +319,8 @@ def bygroups(*args):
                 if data is not None:
                     if ctx:
                         ctx.pos = match.start(i + 1)
-                    for item in action(lexer, _PseudoMatch(match.start(i + 1),
-                                       data), ctx):
+                    for item in action(lexer,
+                                       _PseudoMatch(match.start(i + 1), data), ctx):
                         if item:
                             yield item
         if ctx:
@@ -655,6 +654,8 @@ class RegexLexer(Lexer):
                         statetokens = tokendefs[statestack[-1]]
                     break
             else:
+                # We are here only if all state tokens have been considered
+                # and there was not a match on any of them.
                 try:
                     if text[pos] == '\n':
                         # at EOL, reset state to "root"

packages/wakatime/packages/pygments/lexers/__init__.py

@@ -5,7 +5,7 @@
 
     Pygments lexers.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -22,7 +22,7 @@ from pygments.util import ClassNotFound, itervalues, guess_decode
 
 
 __all__ = ['get_lexer_by_name', 'get_lexer_for_filename', 'find_lexer_class',
-           'guess_lexer'] + list(LEXERS)
+           'guess_lexer', 'load_lexer_from_file'] + list(LEXERS)
 
 _lexer_cache = {}
 _pattern_cache = {}
@@ -72,6 +72,28 @@ def find_lexer_class(name):
             return cls
 
 
+def find_lexer_class_by_name(_alias):
+    """Lookup a lexer class by alias.
+
+    Like `get_lexer_by_name`, but does not instantiate the class.
+
+    .. versionadded:: 2.2
+    """
+    if not _alias:
+        raise ClassNotFound('no lexer for alias %r found' % _alias)
+    # lookup builtin lexers
+    for module_name, name, aliases, _, _ in itervalues(LEXERS):
+        if _alias.lower() in aliases:
+            if name not in _lexer_cache:
+                _load_lexers(module_name)
+            return _lexer_cache[name]
+    # continue with lexers from setuptools entrypoints
+    for cls in find_plugin_lexers():
+        if _alias.lower() in cls.aliases:
+            return cls
+    raise ClassNotFound('no lexer for alias %r found' % _alias)
+
+
 def get_lexer_by_name(_alias, **options):
     """Get a lexer by an alias.
 
@@ -88,11 +110,45 @@ def get_lexer_by_name(_alias, **options):
             return _lexer_cache[name](**options)
     # continue with lexers from setuptools entrypoints
     for cls in find_plugin_lexers():
-        if _alias in cls.aliases:
+        if _alias.lower() in cls.aliases:
             return cls(**options)
     raise ClassNotFound('no lexer for alias %r found' % _alias)
 
 
+def load_lexer_from_file(filename, lexername="CustomLexer", **options):
+    """Load a lexer from a file.
+
+    This method expects a file located relative to the current working
+    directory, which contains a Lexer class. By default, it expects the
+    Lexer to be name CustomLexer; you can specify your own class name
+    as the second argument to this function.
+
+    Users should be very careful with the input, because this method
+    is equivalent to running eval on the input file.
+
+    Raises ClassNotFound if there are any problems importing the Lexer.
+
+    .. versionadded:: 2.2
+    """
+    try:
+        # This empty dict will contain the namespace for the exec'd file
+        custom_namespace = {}
+        exec(open(filename, 'rb').read(), custom_namespace)
+        # Retrieve the class `lexername` from that namespace
+        if lexername not in custom_namespace:
+            raise ClassNotFound('no valid %s class found in %s' %
+                                (lexername, filename))
+        lexer_class = custom_namespace[lexername]
+        # And finally instantiate it with the options
+        return lexer_class(**options)
+    except IOError as err:
+        raise ClassNotFound('cannot read %s' % filename)
+    except ClassNotFound as err:
+        raise
+    except Exception as err:
+        raise ClassNotFound('error when loading custom lexer: %s' % err)
+
+
 def find_lexer_class_for_filename(_fn, code=None):
     """Get a lexer for a filename.
 
@@ -127,8 +183,8 @@ def find_lexer_class_for_filename(_fn, code=None):
         # gets turned into 0.0.  Run scripts/detect_missing_analyse_text.py
         # to find lexers which need it overridden.
         if code:
-            return cls.analyse_text(code) + bonus
-        return cls.priority + bonus
+            return cls.analyse_text(code) + bonus, cls.__name__
+        return cls.priority + bonus, cls.__name__
 
     if matches:
         matches.sort(key=get_rating)

packages/wakatime/packages/pygments/lexers/_asy_builtins.py

@@ -10,7 +10,7 @@
     TODO: perl/python script in Asymptote SVN similar to asy-list.pl but only
     for function and variable names.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/_cl_builtins.py

@@ -5,7 +5,7 @@
 
     ANSI Common Lisp builtins.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/_lua_builtins.py

@@ -9,60 +9,71 @@
 
     Do not edit the MODULES dict by hand.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 from __future__ import print_function
 
-
 MODULES = {'basic': ('_G',
            '_VERSION',
            'assert',
            'collectgarbage',
            'dofile',
            'error',
-           'getfenv',
            'getmetatable',
            'ipairs',
            'load',
            'loadfile',
-           'loadstring',
            'next',
            'pairs',
            'pcall',
            'print',
            'rawequal',
            'rawget',
+           'rawlen',
            'rawset',
            'select',
-           'setfenv',
            'setmetatable',
            'tonumber',
            'tostring',
            'type',
-           'unpack',
            'xpcall'),
+ 'bit32': ('bit32.arshift',
+           'bit32.band',
+           'bit32.bnot',
+           'bit32.bor',
+           'bit32.btest',
+           'bit32.bxor',
+           'bit32.extract',
+           'bit32.lrotate',
+           'bit32.lshift',
+           'bit32.replace',
+           'bit32.rrotate',
+           'bit32.rshift'),
  'coroutine': ('coroutine.create',
+               'coroutine.isyieldable',
                'coroutine.resume',
                'coroutine.running',
                'coroutine.status',
                'coroutine.wrap',
                'coroutine.yield'),
  'debug': ('debug.debug',
-           'debug.getfenv',
            'debug.gethook',
            'debug.getinfo',
            'debug.getlocal',
            'debug.getmetatable',
            'debug.getregistry',
            'debug.getupvalue',
-           'debug.setfenv',
+           'debug.getuservalue',
            'debug.sethook',
            'debug.setlocal',
            'debug.setmetatable',
            'debug.setupvalue',
-           'debug.traceback'),
+           'debug.setuservalue',
+           'debug.traceback',
+           'debug.upvalueid',
+           'debug.upvaluejoin'),
  'io': ('io.close',
         'io.flush',
         'io.input',
@@ -71,17 +82,20 @@ MODULES = {'basic': ('_G',
         'io.output',
         'io.popen',
         'io.read',
+        'io.stderr',
+        'io.stdin',
+        'io.stdout',
         'io.tmpfile',
         'io.type',
         'io.write'),
  'math': ('math.abs',
           'math.acos',
           'math.asin',
-          'math.atan2',
           'math.atan',
+          'math.atan2',
           'math.ceil',
-          'math.cosh',
           'math.cos',
+          'math.cosh',
           'math.deg',
           'math.exp',
           'math.floor',
@@ -89,29 +103,34 @@ MODULES = {'basic': ('_G',
           'math.frexp',
           'math.huge',
           'math.ldexp',
-          'math.log10',
           'math.log',
           'math.max',
+          'math.maxinteger',
           'math.min',
+          'math.mininteger',
           'math.modf',
           'math.pi',
           'math.pow',
           'math.rad',
           'math.random',
           'math.randomseed',
-          'math.sinh',
           'math.sin',
+          'math.sinh',
           'math.sqrt',
+          'math.tan',
           'math.tanh',
-          'math.tan'),
- 'modules': ('module',
-             'require',
+          'math.tointeger',
+          'math.type',
+          'math.ult'),
+ 'modules': ('package.config',
              'package.cpath',
              'package.loaded',
              'package.loadlib',
              'package.path',
              'package.preload',
-             'package.seeall'),
+             'package.searchers',
+             'package.searchpath',
+             'require'),
  'os': ('os.clock',
         'os.date',
         'os.difftime',
@@ -133,18 +152,37 @@ MODULES = {'basic': ('_G',
             'string.len',
             'string.lower',
             'string.match',
+            'string.pack',
+            'string.packsize',
             'string.rep',
             'string.reverse',
             'string.sub',
+            'string.unpack',
             'string.upper'),
  'table': ('table.concat',
            'table.insert',
-           'table.maxn',
+           'table.move',
+           'table.pack',
            'table.remove',
-           'table.sort')}
+           'table.sort',
+           'table.unpack'),
+ 'utf8': ('utf8.char',
+          'utf8.charpattern',
+          'utf8.codepoint',
+          'utf8.codes',
+          'utf8.len',
+          'utf8.offset')}
 
-if __name__ == '__main__':
+if __name__ == '__main__':  # pragma: no cover
     import re
+    import sys
+
+    # urllib ends up wanting to import a module called 'math' -- if
+    # pygments/lexers is in the path, this ends badly.
+    for i in range(len(sys.path)-1, -1, -1):
+        if sys.path[i].endswith('/lexers'):
+            del sys.path[i]
+
     try:
         from urllib import urlopen
     except ImportError:
@@ -195,7 +233,7 @@ if __name__ == '__main__':
 
     def get_newest_version():
         f = urlopen('http://www.lua.org/manual/')
-        r = re.compile(r'^<A HREF="(\d\.\d)/">Lua \1</A>')
+        r = re.compile(r'^<A HREF="(\d\.\d)/">(Lua )?\1</A>')
         for line in f:
             m = r.match(line)
             if m is not None:
@@ -203,7 +241,7 @@ if __name__ == '__main__':
 
     def get_lua_functions(version):
         f = urlopen('http://www.lua.org/manual/%s/' % version)
-        r = re.compile(r'^<A HREF="manual.html#pdf-(.+)">\1</A>')
+        r = re.compile(r'^<A HREF="manual.html#pdf-(?!lua|LUA)([^:]+)">\1</A>')
         functions = []
         for line in f:
             m = r.match(line)
@@ -235,15 +273,22 @@ if __name__ == '__main__':
 
     def run():
         version = get_newest_version()
-        print('> Downloading function index for Lua %s' % version)
-        functions = get_lua_functions(version)
-        print('> %d functions found:' % len(functions))
+        functions = set()
+        for v in ('5.2', version):
+            print('> Downloading function index for Lua %s' % v)
+            f = get_lua_functions(v)
+            print('> %d functions found, %d new:' %
+                  (len(f), len(set(f) - functions)))
+            functions |= set(f)
+
+        functions = sorted(functions)
 
         modules = {}
         for full_function_name in functions:
             print('>> %s' % full_function_name)
             m = get_function_module(full_function_name)
             modules.setdefault(m, []).append(full_function_name)
+        modules = {k: tuple(v) for k, v in modules.iteritems()}
 
         regenerate(__file__, modules)
 

packages/wakatime/packages/pygments/lexers/_mapping.py

@@ -9,21 +9,27 @@
 
     Do not alter the LEXERS dictionary by hand.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2014, 2016 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 from __future__ import print_function
 
 LEXERS = {
-    'ABAPLexer': ('pygments.lexers.business', 'ABAP', ('abap',), ('*.abap',), ('text/x-abap',)),
+    'ABAPLexer': ('pygments.lexers.business', 'ABAP', ('abap',), ('*.abap', '*.ABAP'), ('text/x-abap',)),
     'APLLexer': ('pygments.lexers.apl', 'APL', ('apl',), ('*.apl',), ()),
+    'AbnfLexer': ('pygments.lexers.grammar_notation', 'ABNF', ('abnf',), ('*.abnf',), ('text/x-abnf',)),
     'ActionScript3Lexer': ('pygments.lexers.actionscript', 'ActionScript 3', ('as3', 'actionscript3'), ('*.as',), ('application/x-actionscript3', 'text/x-actionscript3', 'text/actionscript3')),
     'ActionScriptLexer': ('pygments.lexers.actionscript', 'ActionScript', ('as', 'actionscript'), ('*.as',), ('application/x-actionscript', 'text/x-actionscript', 'text/actionscript')),
     'AdaLexer': ('pygments.lexers.pascal', 'Ada', ('ada', 'ada95', 'ada2005'), ('*.adb', '*.ads', '*.ada'), ('text/x-ada',)),
+    'AdlLexer': ('pygments.lexers.archetype', 'ADL', ('adl',), ('*.adl', '*.adls', '*.adlf', '*.adlx'), ()),
     'AgdaLexer': ('pygments.lexers.haskell', 'Agda', ('agda',), ('*.agda',), ('text/x-agda',)),
+    'AheuiLexer': ('pygments.lexers.esoteric', 'Aheui', ('aheui',), ('*.aheui',), ()),
     'AlloyLexer': ('pygments.lexers.dsls', 'Alloy', ('alloy',), ('*.als',), ('text/x-alloy',)),
     'AmbientTalkLexer': ('pygments.lexers.ambient', 'AmbientTalk', ('at', 'ambienttalk', 'ambienttalk/2'), ('*.at',), ('text/x-ambienttalk',)),
+    'AmplLexer': ('pygments.lexers.ampl', 'Ampl', ('ampl',), ('*.run',), ()),
+    'Angular2HtmlLexer': ('pygments.lexers.templates', 'HTML + Angular2', ('html+ng2',), ('*.ng2',), ()),
+    'Angular2Lexer': ('pygments.lexers.templates', 'Angular2', ('ng2',), (), ()),
     'AntlrActionScriptLexer': ('pygments.lexers.parsers', 'ANTLR With ActionScript Target', ('antlr-as', 'antlr-actionscript'), ('*.G', '*.g'), ()),
     'AntlrCSharpLexer': ('pygments.lexers.parsers', 'ANTLR With C# Target', ('antlr-csharp', 'antlr-c#'), ('*.G', '*.g'), ()),
     'AntlrCppLexer': ('pygments.lexers.parsers', 'ANTLR With CPP Target', ('antlr-cpp',), ('*.G', '*.g'), ()),
@@ -35,29 +41,40 @@ LEXERS = {
     'AntlrRubyLexer': ('pygments.lexers.parsers', 'ANTLR With Ruby Target', ('antlr-ruby', 'antlr-rb'), ('*.G', '*.g'), ()),
     'ApacheConfLexer': ('pygments.lexers.configs', 'ApacheConf', ('apacheconf', 'aconf', 'apache'), ('.htaccess', 'apache.conf', 'apache2.conf'), ('text/x-apacheconf',)),
     'AppleScriptLexer': ('pygments.lexers.scripting', 'AppleScript', ('applescript',), ('*.applescript',), ()),
+    'ArduinoLexer': ('pygments.lexers.c_like', 'Arduino', ('arduino',), ('*.ino',), ('text/x-arduino',)),
     'AspectJLexer': ('pygments.lexers.jvm', 'AspectJ', ('aspectj',), ('*.aj',), ('text/x-aspectj',)),
     'AsymptoteLexer': ('pygments.lexers.graphics', 'Asymptote', ('asy', 'asymptote'), ('*.asy',), ('text/x-asymptote',)),
     'AutoItLexer': ('pygments.lexers.automation', 'AutoIt', ('autoit',), ('*.au3',), ('text/x-autoit',)),
     'AutohotkeyLexer': ('pygments.lexers.automation', 'autohotkey', ('ahk', 'autohotkey'), ('*.ahk', '*.ahkl'), ('text/x-autohotkey',)),
     'AwkLexer': ('pygments.lexers.textedit', 'Awk', ('awk', 'gawk', 'mawk', 'nawk'), ('*.awk',), ('application/x-awk',)),
     'BBCodeLexer': ('pygments.lexers.markup', 'BBCode', ('bbcode',), (), ('text/x-bbcode',)),
+    'BCLexer': ('pygments.lexers.algebra', 'BC', ('bc',), ('*.bc',), ()),
+    'BSTLexer': ('pygments.lexers.bibtex', 'BST', ('bst', 'bst-pybtex'), ('*.bst',), ()),
     'BaseMakefileLexer': ('pygments.lexers.make', 'Base Makefile', ('basemake',), (), ()),
-    'BashLexer': ('pygments.lexers.shell', 'Bash', ('bash', 'sh', 'ksh', 'shell'), ('*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass', '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'PKGBUILD'), ('application/x-sh', 'application/x-shellscript')),
-    'BashSessionLexer': ('pygments.lexers.shell', 'Bash Session', ('console',), ('*.sh-session',), ('application/x-shell-session',)),
+    'BashLexer': ('pygments.lexers.shell', 'Bash', ('bash', 'sh', 'ksh', 'zsh', 'shell'), ('*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass', '*.exheres-0', '*.exlib', '*.zsh', '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'zshrc', '.zshrc', 'PKGBUILD'), ('application/x-sh', 'application/x-shellscript')),
+    'BashSessionLexer': ('pygments.lexers.shell', 'Bash Session', ('console', 'shell-session'), ('*.sh-session', '*.shell-session'), ('application/x-shell-session', 'application/x-sh-session')),
     'BatchLexer': ('pygments.lexers.shell', 'Batchfile', ('bat', 'batch', 'dosbatch', 'winbatch'), ('*.bat', '*.cmd'), ('application/x-dos-batch',)),
     'BefungeLexer': ('pygments.lexers.esoteric', 'Befunge', ('befunge',), ('*.befunge',), ('application/x-befunge',)),
+    'BibTeXLexer': ('pygments.lexers.bibtex', 'BibTeX', ('bib', 'bibtex'), ('*.bib',), ('text/x-bibtex',)),
     'BlitzBasicLexer': ('pygments.lexers.basic', 'BlitzBasic', ('blitzbasic', 'b3d', 'bplus'), ('*.bb', '*.decls'), ('text/x-bb',)),
     'BlitzMaxLexer': ('pygments.lexers.basic', 'BlitzMax', ('blitzmax', 'bmax'), ('*.bmx',), ('text/x-bmx',)),
+    'BnfLexer': ('pygments.lexers.grammar_notation', 'BNF', ('bnf',), ('*.bnf',), ('text/x-bnf',)),
     'BooLexer': ('pygments.lexers.dotnet', 'Boo', ('boo',), ('*.boo',), ('text/x-boo',)),
+    'BoogieLexer': ('pygments.lexers.verification', 'Boogie', ('boogie',), ('*.bpl',), ()),
     'BrainfuckLexer': ('pygments.lexers.esoteric', 'Brainfuck', ('brainfuck', 'bf'), ('*.bf', '*.b'), ('application/x-brainfuck',)),
     'BroLexer': ('pygments.lexers.dsls', 'Bro', ('bro',), ('*.bro',), ()),
     'BugsLexer': ('pygments.lexers.modeling', 'BUGS', ('bugs', 'winbugs', 'openbugs'), ('*.bug',), ()),
+    'CAmkESLexer': ('pygments.lexers.esoteric', 'CAmkES', ('camkes', 'idl4'), ('*.camkes', '*.idl4'), ()),
     'CLexer': ('pygments.lexers.c_cpp', 'C', ('c',), ('*.c', '*.h', '*.idc'), ('text/x-chdr', 'text/x-csrc')),
     'CMakeLexer': ('pygments.lexers.make', 'CMake', ('cmake',), ('*.cmake', 'CMakeLists.txt'), ('text/x-cmake',)),
     'CObjdumpLexer': ('pygments.lexers.asm', 'c-objdump', ('c-objdump',), ('*.c-objdump',), ('text/x-c-objdump',)),
+    'CPSALexer': ('pygments.lexers.lisp', 'CPSA', ('cpsa',), ('*.cpsa',), ()),
     'CSharpAspxLexer': ('pygments.lexers.dotnet', 'aspx-cs', ('aspx-cs',), ('*.aspx', '*.asax', '*.ascx', '*.ashx', '*.asmx', '*.axd'), ()),
     'CSharpLexer': ('pygments.lexers.dotnet', 'C#', ('csharp', 'c#'), ('*.cs',), ('text/x-csharp',)),
     'Ca65Lexer': ('pygments.lexers.asm', 'ca65 assembler', ('ca65',), ('*.s',), ()),
+    'CadlLexer': ('pygments.lexers.archetype', 'cADL', ('cadl',), ('*.cadl',), ()),
+    'CapDLLexer': ('pygments.lexers.esoteric', 'CapDL', ('capdl',), ('*.cdl',), ()),
+    'CapnProtoLexer': ('pygments.lexers.capnproto', "Cap'n Proto", ('capnp',), ('*.capnp',), ()),
     'CbmBasicV2Lexer': ('pygments.lexers.basic', 'CBM BASIC V2', ('cbmbas',), ('*.bas',), ()),
     'CeylonLexer': ('pygments.lexers.jvm', 'Ceylon', ('ceylon',), ('*.ceylon',), ('text/x-ceylon',)),
     'Cfengine3Lexer': ('pygments.lexers.configs', 'CFEngine3', ('cfengine3', 'cf3'), ('*.cf',), ()),
@@ -67,8 +84,9 @@ LEXERS = {
     'CheetahJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Cheetah', ('js+cheetah', 'javascript+cheetah', 'js+spitfire', 'javascript+spitfire'), (), ('application/x-javascript+cheetah', 'text/x-javascript+cheetah', 'text/javascript+cheetah', 'application/x-javascript+spitfire', 'text/x-javascript+spitfire', 'text/javascript+spitfire')),
     'CheetahLexer': ('pygments.lexers.templates', 'Cheetah', ('cheetah', 'spitfire'), ('*.tmpl', '*.spt'), ('application/x-cheetah', 'application/x-spitfire')),
     'CheetahXmlLexer': ('pygments.lexers.templates', 'XML+Cheetah', ('xml+cheetah', 'xml+spitfire'), (), ('application/xml+cheetah', 'application/xml+spitfire')),
-    'CirruLexer': ('pygments.lexers.webmisc', 'Cirru', ('cirru',), ('*.cirru', '*.cr'), ('text/x-cirru',)),
+    'CirruLexer': ('pygments.lexers.webmisc', 'Cirru', ('cirru',), ('*.cirru',), ('text/x-cirru',)),
     'ClayLexer': ('pygments.lexers.c_like', 'Clay', ('clay',), ('*.clay',), ('text/x-clay',)),
+    'CleanLexer': ('pygments.lexers.clean', 'Clean', ('clean',), ('*.icl', '*.dcl'), ()),
     'ClojureLexer': ('pygments.lexers.jvm', 'Clojure', ('clojure', 'clj'), ('*.clj',), ('text/x-clojure', 'application/x-clojure')),
     'ClojureScriptLexer': ('pygments.lexers.jvm', 'ClojureScript', ('clojurescript', 'cljs'), ('*.cljs',), ('text/x-clojurescript', 'application/x-clojurescript')),
     'CobolFreeformatLexer': ('pygments.lexers.business', 'COBOLFree', ('cobolfree',), ('*.cbl', '*.CBL'), ()),
@@ -77,12 +95,18 @@ LEXERS = {
     'ColdfusionCFCLexer': ('pygments.lexers.templates', 'Coldfusion CFC', ('cfc',), ('*.cfc',), ()),
     'ColdfusionHtmlLexer': ('pygments.lexers.templates', 'Coldfusion HTML', ('cfm',), ('*.cfm', '*.cfml'), ('application/x-coldfusion',)),
     'ColdfusionLexer': ('pygments.lexers.templates', 'cfstatement', ('cfs',), (), ()),
-    'CommonLispLexer': ('pygments.lexers.lisp', 'Common Lisp', ('common-lisp', 'cl', 'lisp', 'elisp', 'emacs', 'emacs-lisp'), ('*.cl', '*.lisp', '*.el'), ('text/x-common-lisp',)),
+    'CommonLispLexer': ('pygments.lexers.lisp', 'Common Lisp', ('common-lisp', 'cl', 'lisp'), ('*.cl', '*.lisp'), ('text/x-common-lisp',)),
+    'ComponentPascalLexer': ('pygments.lexers.oberon', 'Component Pascal', ('componentpascal', 'cp'), ('*.cp', '*.cps'), ('text/x-component-pascal',)),
     'CoqLexer': ('pygments.lexers.theorem', 'Coq', ('coq',), ('*.v',), ('text/x-coq',)),
     'CppLexer': ('pygments.lexers.c_cpp', 'C++', ('cpp', 'c++'), ('*.cpp', '*.hpp', '*.c++', '*.h++', '*.cc', '*.hh', '*.cxx', '*.hxx', '*.C', '*.H', '*.cp', '*.CPP'), ('text/x-c++hdr', 'text/x-c++src')),
     'CppObjdumpLexer': ('pygments.lexers.asm', 'cpp-objdump', ('cpp-objdump', 'c++-objdumb', 'cxx-objdump'), ('*.cpp-objdump', '*.c++-objdump', '*.cxx-objdump'), ('text/x-cpp-objdump',)),
+    'CrmshLexer': ('pygments.lexers.dsls', 'Crmsh', ('crmsh', 'pcmk'), ('*.crmsh', '*.pcmk'), ()),
     'CrocLexer': ('pygments.lexers.d', 'Croc', ('croc',), ('*.croc',), ('text/x-crocsrc',)),
     'CryptolLexer': ('pygments.lexers.haskell', 'Cryptol', ('cryptol', 'cry'), ('*.cry',), ('text/x-cryptol',)),
+    'CrystalLexer': ('pygments.lexers.crystal', 'Crystal', ('cr', 'crystal'), ('*.cr',), ('text/x-crystal',)),
+    'CsoundDocumentLexer': ('pygments.lexers.csound', 'Csound Document', ('csound-document', 'csound-csd'), ('*.csd',), ()),
+    'CsoundOrchestraLexer': ('pygments.lexers.csound', 'Csound Orchestra', ('csound', 'csound-orc'), ('*.orc',), ()),
+    'CsoundScoreLexer': ('pygments.lexers.csound', 'Csound Score', ('csound-score', 'csound-sco'), ('*.sco',), ()),
     'CssDjangoLexer': ('pygments.lexers.templates', 'CSS+Django/Jinja', ('css+django', 'css+jinja'), (), ('text/css+django', 'text/css+jinja')),
     'CssErbLexer': ('pygments.lexers.templates', 'CSS+Ruby', ('css+erb', 'css+ruby'), (), ('text/css+ruby',)),
     'CssGenshiLexer': ('pygments.lexers.templates', 'CSS+Genshi Text', ('css+genshitext', 'css+genshi'), (), ('text/css+genshi',)),
@@ -97,7 +121,7 @@ LEXERS = {
     'DarcsPatchLexer': ('pygments.lexers.diff', 'Darcs Patch', ('dpatch',), ('*.dpatch', '*.darcspatch'), ()),
     'DartLexer': ('pygments.lexers.javascript', 'Dart', ('dart',), ('*.dart',), ('text/x-dart',)),
     'DebianControlLexer': ('pygments.lexers.installers', 'Debian Control file', ('control', 'debcontrol'), ('control',), ()),
-    'DelphiLexer': ('pygments.lexers.pascal', 'Delphi', ('delphi', 'pas', 'pascal', 'objectpascal'), ('*.pas',), ('text/x-pascal',)),
+    'DelphiLexer': ('pygments.lexers.pascal', 'Delphi', ('delphi', 'pas', 'pascal', 'objectpascal'), ('*.pas', '*.dpr'), ('text/x-pascal',)),
     'DgLexer': ('pygments.lexers.python', 'dg', ('dg',), ('*.dg',), ('text/x-dg',)),
     'DiffLexer': ('pygments.lexers.diff', 'Diff', ('diff', 'udiff'), ('*.diff', '*.patch'), ('text/x-diff', 'text/x-patch')),
     'DjangoLexer': ('pygments.lexers.templates', 'Django/Jinja', ('django', 'jinja'), (), ('application/x-django-templating', 'application/x-jinja')),
@@ -109,22 +133,31 @@ LEXERS = {
     'DylanLidLexer': ('pygments.lexers.dylan', 'DylanLID', ('dylan-lid', 'lid'), ('*.lid', '*.hdp'), ('text/x-dylan-lid',)),
     'ECLLexer': ('pygments.lexers.ecl', 'ECL', ('ecl',), ('*.ecl',), ('application/x-ecl',)),
     'ECLexer': ('pygments.lexers.c_like', 'eC', ('ec',), ('*.ec', '*.eh'), ('text/x-echdr', 'text/x-ecsrc')),
+    'EarlGreyLexer': ('pygments.lexers.javascript', 'Earl Grey', ('earl-grey', 'earlgrey', 'eg'), ('*.eg',), ('text/x-earl-grey',)),
+    'EasytrieveLexer': ('pygments.lexers.scripting', 'Easytrieve', ('easytrieve',), ('*.ezt', '*.mac'), ('text/x-easytrieve',)),
     'EbnfLexer': ('pygments.lexers.parsers', 'EBNF', ('ebnf',), ('*.ebnf',), ('text/x-ebnf',)),
     'EiffelLexer': ('pygments.lexers.eiffel', 'Eiffel', ('eiffel',), ('*.e',), ('text/x-eiffel',)),
     'ElixirConsoleLexer': ('pygments.lexers.erlang', 'Elixir iex session', ('iex',), (), ('text/x-elixir-shellsession',)),
     'ElixirLexer': ('pygments.lexers.erlang', 'Elixir', ('elixir', 'ex', 'exs'), ('*.ex', '*.exs'), ('text/x-elixir',)),
+    'ElmLexer': ('pygments.lexers.elm', 'Elm', ('elm',), ('*.elm',), ('text/x-elm',)),
+    'EmacsLispLexer': ('pygments.lexers.lisp', 'EmacsLisp', ('emacs', 'elisp', 'emacs-lisp'), ('*.el',), ('text/x-elisp', 'application/x-elisp')),
     'ErbLexer': ('pygments.lexers.templates', 'ERB', ('erb',), (), ('application/x-ruby-templating',)),
     'ErlangLexer': ('pygments.lexers.erlang', 'Erlang', ('erlang',), ('*.erl', '*.hrl', '*.es', '*.escript'), ('text/x-erlang',)),
     'ErlangShellLexer': ('pygments.lexers.erlang', 'Erlang erl session', ('erl',), ('*.erl-sh',), ('text/x-erl-shellsession',)),
     'EvoqueHtmlLexer': ('pygments.lexers.templates', 'HTML+Evoque', ('html+evoque',), ('*.html',), ('text/html+evoque',)),
     'EvoqueLexer': ('pygments.lexers.templates', 'Evoque', ('evoque',), ('*.evoque',), ('application/x-evoque',)),
     'EvoqueXmlLexer': ('pygments.lexers.templates', 'XML+Evoque', ('xml+evoque',), ('*.xml',), ('application/xml+evoque',)),
-    'FSharpLexer': ('pygments.lexers.dotnet', 'FSharp', ('fsharp',), ('*.fs', '*.fsi'), ('text/x-fsharp',)),
+    'EzhilLexer': ('pygments.lexers.ezhil', 'Ezhil', ('ezhil',), ('*.n',), ('text/x-ezhil',)),
+    'FSharpLexer': ('pygments.lexers.dotnet', 'F#', ('fsharp',), ('*.fs', '*.fsi'), ('text/x-fsharp',)),
     'FactorLexer': ('pygments.lexers.factor', 'Factor', ('factor',), ('*.factor',), ('text/x-factor',)),
     'FancyLexer': ('pygments.lexers.ruby', 'Fancy', ('fancy', 'fy'), ('*.fy', '*.fancypack'), ('text/x-fancysrc',)),
     'FantomLexer': ('pygments.lexers.fantom', 'Fantom', ('fan',), ('*.fan',), ('application/x-fantom',)),
     'FelixLexer': ('pygments.lexers.felix', 'Felix', ('felix', 'flx'), ('*.flx', '*.flxh'), ('text/x-felix',)),
-    'FortranLexer': ('pygments.lexers.fortran', 'Fortran', ('fortran',), ('*.f', '*.f90', '*.F', '*.F90'), ('text/x-fortran',)),
+    'FishShellLexer': ('pygments.lexers.shell', 'Fish', ('fish', 'fishshell'), ('*.fish', '*.load'), ('application/x-fish',)),
+    'FlatlineLexer': ('pygments.lexers.dsls', 'Flatline', ('flatline',), (), ('text/x-flatline',)),
+    'ForthLexer': ('pygments.lexers.forth', 'Forth', ('forth',), ('*.frt', '*.fs'), ('application/x-forth',)),
+    'FortranFixedLexer': ('pygments.lexers.fortran', 'FortranFixed', ('fortranfixed',), ('*.f', '*.F'), ()),
+    'FortranLexer': ('pygments.lexers.fortran', 'Fortran', ('fortran',), ('*.f03', '*.f90', '*.F03', '*.F90'), ('text/x-fortran',)),
     'FoxProLexer': ('pygments.lexers.foxpro', 'FoxPro', ('foxpro', 'vfp', 'clipper', 'xbase'), ('*.PRG', '*.prg'), ()),
     'GAPLexer': ('pygments.lexers.algebra', 'GAP', ('gap',), ('*.g', '*.gd', '*.gi', '*.gap'), ()),
     'GLShaderLexer': ('pygments.lexers.graphics', 'GLSL', ('glsl',), ('*.vert', '*.frag', '*.geo'), ('text/x-glslsrc',)),
@@ -140,12 +173,14 @@ LEXERS = {
     'GosuLexer': ('pygments.lexers.jvm', 'Gosu', ('gosu',), ('*.gs', '*.gsx', '*.gsp', '*.vark'), ('text/x-gosu',)),
     'GosuTemplateLexer': ('pygments.lexers.jvm', 'Gosu Template', ('gst',), ('*.gst',), ('text/x-gosu-template',)),
     'GroffLexer': ('pygments.lexers.markup', 'Groff', ('groff', 'nroff', 'man'), ('*.[1234567]', '*.man'), ('application/x-troff', 'text/troff')),
-    'GroovyLexer': ('pygments.lexers.jvm', 'Groovy', ('groovy',), ('*.groovy',), ('text/x-groovy',)),
+    'GroovyLexer': ('pygments.lexers.jvm', 'Groovy', ('groovy',), ('*.groovy', '*.gradle'), ('text/x-groovy',)),
     'HamlLexer': ('pygments.lexers.html', 'Haml', ('haml',), ('*.haml',), ('text/x-haml',)),
     'HandlebarsHtmlLexer': ('pygments.lexers.templates', 'HTML+Handlebars', ('html+handlebars',), ('*.handlebars', '*.hbs'), ('text/html+handlebars', 'text/x-handlebars-template')),
     'HandlebarsLexer': ('pygments.lexers.templates', 'Handlebars', ('handlebars',), (), ()),
     'HaskellLexer': ('pygments.lexers.haskell', 'Haskell', ('haskell', 'hs'), ('*.hs',), ('text/x-haskell',)),
     'HaxeLexer': ('pygments.lexers.haxe', 'Haxe', ('hx', 'haxe', 'hxsl'), ('*.hx', '*.hxsl'), ('text/haxe', 'text/x-haxe', 'text/x-hx')),
+    'HexdumpLexer': ('pygments.lexers.hexdump', 'Hexdump', ('hexdump',), (), ()),
+    'HsailLexer': ('pygments.lexers.asm', 'HSAIL', ('hsail', 'hsa'), ('*.hsail',), ('text/x-hsail',)),
     'HtmlDjangoLexer': ('pygments.lexers.templates', 'HTML+Django/Jinja', ('html+django', 'html+jinja', 'htmldjango'), (), ('text/html+django', 'text/html+jinja')),
     'HtmlGenshiLexer': ('pygments.lexers.templates', 'HTML+Genshi', ('html+genshi', 'html+kid'), (), ('text/html+genshi',)),
     'HtmlLexer': ('pygments.lexers.html', 'HTML', ('html',), ('*.html', '*.htm', '*.xhtml', '*.xslt'), ('text/html', 'application/xhtml+xml')),
@@ -161,26 +196,30 @@ LEXERS = {
     'Inform6Lexer': ('pygments.lexers.int_fiction', 'Inform 6', ('inform6', 'i6'), ('*.inf',), ()),
     'Inform6TemplateLexer': ('pygments.lexers.int_fiction', 'Inform 6 template', ('i6t',), ('*.i6t',), ()),
     'Inform7Lexer': ('pygments.lexers.int_fiction', 'Inform 7', ('inform7', 'i7'), ('*.ni', '*.i7x'), ()),
-    'IniLexer': ('pygments.lexers.configs', 'INI', ('ini', 'cfg', 'dosini'), ('*.ini', '*.cfg'), ('text/x-ini',)),
+    'IniLexer': ('pygments.lexers.configs', 'INI', ('ini', 'cfg', 'dosini'), ('*.ini', '*.cfg', '*.inf'), ('text/x-ini', 'text/inf')),
     'IoLexer': ('pygments.lexers.iolang', 'Io', ('io',), ('*.io',), ('text/x-iosrc',)),
     'IokeLexer': ('pygments.lexers.jvm', 'Ioke', ('ioke', 'ik'), ('*.ik',), ('text/x-iokesrc',)),
     'IrcLogsLexer': ('pygments.lexers.textfmts', 'IRC logs', ('irc',), ('*.weechatlog',), ('text/x-irclog',)),
     'IsabelleLexer': ('pygments.lexers.theorem', 'Isabelle', ('isabelle',), ('*.thy',), ('text/x-isabelle',)),
-    'JadeLexer': ('pygments.lexers.html', 'Jade', ('jade',), ('*.jade',), ('text/x-jade',)),
+    'JLexer': ('pygments.lexers.j', 'J', ('j',), ('*.ijs',), ('text/x-j',)),
     'JagsLexer': ('pygments.lexers.modeling', 'JAGS', ('jags',), ('*.jag', '*.bug'), ()),
     'JasminLexer': ('pygments.lexers.jvm', 'Jasmin', ('jasmin', 'jasminxt'), ('*.j',), ()),
     'JavaLexer': ('pygments.lexers.jvm', 'Java', ('java',), ('*.java',), ('text/x-java',)),
     'JavascriptDjangoLexer': ('pygments.lexers.templates', 'JavaScript+Django/Jinja', ('js+django', 'javascript+django', 'js+jinja', 'javascript+jinja'), (), ('application/x-javascript+django', 'application/x-javascript+jinja', 'text/x-javascript+django', 'text/x-javascript+jinja', 'text/javascript+django', 'text/javascript+jinja')),
     'JavascriptErbLexer': ('pygments.lexers.templates', 'JavaScript+Ruby', ('js+erb', 'javascript+erb', 'js+ruby', 'javascript+ruby'), (), ('application/x-javascript+ruby', 'text/x-javascript+ruby', 'text/javascript+ruby')),
     'JavascriptGenshiLexer': ('pygments.lexers.templates', 'JavaScript+Genshi Text', ('js+genshitext', 'js+genshi', 'javascript+genshitext', 'javascript+genshi'), (), ('application/x-javascript+genshi', 'text/x-javascript+genshi', 'text/javascript+genshi')),
-    'JavascriptLexer': ('pygments.lexers.javascript', 'JavaScript', ('js', 'javascript'), ('*.js',), ('application/javascript', 'application/x-javascript', 'text/x-javascript', 'text/javascript')),
+    'JavascriptLexer': ('pygments.lexers.javascript', 'JavaScript', ('js', 'javascript'), ('*.js', '*.jsm'), ('application/javascript', 'application/x-javascript', 'text/x-javascript', 'text/javascript')),
     'JavascriptPhpLexer': ('pygments.lexers.templates', 'JavaScript+PHP', ('js+php', 'javascript+php'), (), ('application/x-javascript+php', 'text/x-javascript+php', 'text/javascript+php')),
     'JavascriptSmartyLexer': ('pygments.lexers.templates', 'JavaScript+Smarty', ('js+smarty', 'javascript+smarty'), (), ('application/x-javascript+smarty', 'text/x-javascript+smarty', 'text/javascript+smarty')),
+    'JclLexer': ('pygments.lexers.scripting', 'JCL', ('jcl',), ('*.jcl',), ('text/x-jcl',)),
+    'JsgfLexer': ('pygments.lexers.grammar_notation', 'JSGF', ('jsgf',), ('*.jsgf',), ('application/jsgf', 'application/x-jsgf', 'text/jsgf')),
+    'JsonBareObjectLexer': ('pygments.lexers.data', 'JSONBareObject', ('json-object',), (), ('application/json-object',)),
     'JsonLdLexer': ('pygments.lexers.data', 'JSON-LD', ('jsonld', 'json-ld'), ('*.jsonld',), ('application/ld+json',)),
     'JsonLexer': ('pygments.lexers.data', 'JSON', ('json',), ('*.json',), ('application/json',)),
     'JspLexer': ('pygments.lexers.templates', 'Java Server Page', ('jsp',), ('*.jsp',), ('application/x-jsp',)),
     'JuliaConsoleLexer': ('pygments.lexers.julia', 'Julia console', ('jlcon',), (), ()),
     'JuliaLexer': ('pygments.lexers.julia', 'Julia', ('julia', 'jl'), ('*.jl',), ('text/x-julia', 'application/x-julia')),
+    'JuttleLexer': ('pygments.lexers.javascript', 'Juttle', ('juttle', 'juttle'), ('*.juttle',), ('application/juttle', 'application/x-juttle', 'text/x-juttle', 'text/juttle')),
     'KalLexer': ('pygments.lexers.javascript', 'Kal', ('kal',), ('*.kal',), ('text/kal', 'application/kal')),
     'KconfigLexer': ('pygments.lexers.configs', 'Kconfig', ('kconfig', 'menuconfig', 'linux-config', 'kernel-config'), ('Kconfig', '*Config.in*', 'external.in*', 'standard-modules.in'), ('text/x-kconfig',)),
     'KokaLexer': ('pygments.lexers.haskell', 'Koka', ('koka',), ('*.kk', '*.kki'), ('text/x-koka',)),
@@ -192,6 +231,7 @@ LEXERS = {
     'LassoLexer': ('pygments.lexers.javascript', 'Lasso', ('lasso', 'lassoscript'), ('*.lasso', '*.lasso[89]'), ('text/x-lasso',)),
     'LassoXmlLexer': ('pygments.lexers.templates', 'XML+Lasso', ('xml+lasso',), (), ('application/xml+lasso',)),
     'LeanLexer': ('pygments.lexers.theorem', 'Lean', ('lean',), ('*.lean',), ('text/x-lean',)),
+    'LessCssLexer': ('pygments.lexers.css', 'LessCss', ('less',), ('*.less',), ('text/x-less-css',)),
     'LighttpdConfLexer': ('pygments.lexers.configs', 'Lighttpd configuration file', ('lighty', 'lighttpd'), (), ('text/x-lighttpd-conf',)),
     'LimboLexer': ('pygments.lexers.inferno', 'Limbo', ('limbo',), ('*.b',), ('text/limbo',)),
     'LiquidLexer': ('pygments.lexers.templates', 'liquid', ('liquid',), ('*.liquid',), ()),
@@ -205,6 +245,7 @@ LEXERS = {
     'LogtalkLexer': ('pygments.lexers.prolog', 'Logtalk', ('logtalk',), ('*.lgt', '*.logtalk'), ('text/x-logtalk',)),
     'LuaLexer': ('pygments.lexers.scripting', 'Lua', ('lua',), ('*.lua', '*.wlua'), ('text/x-lua', 'application/x-lua')),
     'MOOCodeLexer': ('pygments.lexers.scripting', 'MOOCode', ('moocode', 'moo'), ('*.moo',), ('text/x-moocode',)),
+    'MSDOSSessionLexer': ('pygments.lexers.shell', 'MSDOS Session', ('doscon',), (), ()),
     'MakefileLexer': ('pygments.lexers.make', 'Makefile', ('make', 'makefile', 'mf', 'bsdmake'), ('*.mak', '*.mk', 'Makefile', 'makefile', 'Makefile.*', 'GNUmakefile'), ('text/x-makefile',)),
     'MakoCssLexer': ('pygments.lexers.templates', 'CSS+Mako', ('css+mako',), (), ('text/css+mako',)),
     'MakoHtmlLexer': ('pygments.lexers.templates', 'HTML+Mako', ('html+mako',), (), ('text/html+mako',)),
@@ -212,6 +253,7 @@ LEXERS = {
     'MakoLexer': ('pygments.lexers.templates', 'Mako', ('mako',), ('*.mao',), ('application/x-mako',)),
     'MakoXmlLexer': ('pygments.lexers.templates', 'XML+Mako', ('xml+mako',), (), ('application/xml+mako',)),
     'MaqlLexer': ('pygments.lexers.business', 'MAQL', ('maql',), ('*.maql',), ('text/x-gooddata-maql', 'application/x-gooddata-maql')),
+    'MarkdownLexer': ('pygments.lexers.markup', 'markdown', ('md',), ('*.md',), ('text/x-markdown',)),
     'MaskLexer': ('pygments.lexers.javascript', 'Mask', ('mask',), ('*.mask',), ('text/x-mask',)),
     'MasonLexer': ('pygments.lexers.templates', 'Mason', ('mason',), ('*.m', '*.mhtml', '*.mc', '*.mi', 'autohandler', 'dhandler'), ('application/x-mason',)),
     'MathematicaLexer': ('pygments.lexers.algebra', 'Mathematica', ('mathematica', 'mma', 'nb'), ('*.nb', '*.cdf', '*.nbp', '*.ma'), ('application/mathematica', 'application/vnd.wolfram.mathematica', 'application/vnd.wolfram.mathematica.package', 'application/vnd.wolfram.cdf')),
@@ -219,9 +261,10 @@ LEXERS = {
     'MatlabSessionLexer': ('pygments.lexers.matlab', 'Matlab session', ('matlabsession',), (), ()),
     'MiniDLexer': ('pygments.lexers.d', 'MiniD', ('minid',), (), ('text/x-minidsrc',)),
     'ModelicaLexer': ('pygments.lexers.modeling', 'Modelica', ('modelica',), ('*.mo',), ('text/x-modelica',)),
-    'Modula2Lexer': ('pygments.lexers.pascal', 'Modula-2', ('modula2', 'm2'), ('*.def', '*.mod'), ('text/x-modula2',)),
+    'Modula2Lexer': ('pygments.lexers.modula2', 'Modula-2', ('modula2', 'm2'), ('*.def', '*.mod'), ('text/x-modula2',)),
     'MoinWikiLexer': ('pygments.lexers.markup', 'MoinMoin/Trac Wiki markup', ('trac-wiki', 'moin'), (), ('text/x-trac-wiki',)),
     'MonkeyLexer': ('pygments.lexers.basic', 'Monkey', ('monkey',), ('*.monkey',), ('text/x-monkey',)),
+    'MonteLexer': ('pygments.lexers.monte', 'Monte', ('monte',), ('*.mt',), ()),
     'MoonScriptLexer': ('pygments.lexers.scripting', 'MoonScript', ('moon', 'moonscript'), ('*.moon',), ('text/x-moonscript', 'application/x-moonscript')),
     'MozPreprocCssLexer': ('pygments.lexers.markup', 'CSS+mozpreproc', ('css+mozpreproc',), ('*.css.in',), ()),
     'MozPreprocHashLexer': ('pygments.lexers.markup', 'mozhashpreproc', ('mozhashpreproc',), (), ()),
@@ -238,17 +281,19 @@ LEXERS = {
     'MyghtyJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Myghty', ('js+myghty', 'javascript+myghty'), (), ('application/x-javascript+myghty', 'text/x-javascript+myghty', 'text/javascript+mygthy')),
     'MyghtyLexer': ('pygments.lexers.templates', 'Myghty', ('myghty',), ('*.myt', 'autodelegate'), ('application/x-myghty',)),
     'MyghtyXmlLexer': ('pygments.lexers.templates', 'XML+Myghty', ('xml+myghty',), (), ('application/xml+myghty',)),
+    'NCLLexer': ('pygments.lexers.ncl', 'NCL', ('ncl',), ('*.ncl',), ('text/ncl',)),
     'NSISLexer': ('pygments.lexers.installers', 'NSIS', ('nsis', 'nsi', 'nsh'), ('*.nsi', '*.nsh'), ('text/x-nsis',)),
     'NasmLexer': ('pygments.lexers.asm', 'NASM', ('nasm',), ('*.asm', '*.ASM'), ('text/x-nasm',)),
     'NasmObjdumpLexer': ('pygments.lexers.asm', 'objdump-nasm', ('objdump-nasm',), ('*.objdump-intel',), ('text/x-nasm-objdump',)),
     'NemerleLexer': ('pygments.lexers.dotnet', 'Nemerle', ('nemerle',), ('*.n',), ('text/x-nemerle',)),
     'NesCLexer': ('pygments.lexers.c_like', 'nesC', ('nesc',), ('*.nc',), ('text/x-nescsrc',)),
-    'NewLispLexer': ('pygments.lexers.lisp', 'NewLisp', ('newlisp',), ('*.lsp', '*.nl'), ('text/x-newlisp', 'application/x-newlisp')),
+    'NewLispLexer': ('pygments.lexers.lisp', 'NewLisp', ('newlisp',), ('*.lsp', '*.nl', '*.kif'), ('text/x-newlisp', 'application/x-newlisp')),
     'NewspeakLexer': ('pygments.lexers.smalltalk', 'Newspeak', ('newspeak',), ('*.ns2',), ('text/x-newspeak',)),
-    'NginxConfLexer': ('pygments.lexers.configs', 'Nginx configuration file', ('nginx',), (), ('text/x-nginx-conf',)),
-    'NimrodLexer': ('pygments.lexers.nimrod', 'Nimrod', ('nimrod', 'nim'), ('*.nim', '*.nimrod'), ('text/x-nimrod',)),
+    'NginxConfLexer': ('pygments.lexers.configs', 'Nginx configuration file', ('nginx',), ('nginx.conf',), ('text/x-nginx-conf',)),
+    'NimrodLexer': ('pygments.lexers.nimrod', 'Nimrod', ('nim', 'nimrod'), ('*.nim', '*.nimrod'), ('text/x-nim',)),
     'NitLexer': ('pygments.lexers.nit', 'Nit', ('nit',), ('*.nit',), ()),
     'NixLexer': ('pygments.lexers.nix', 'Nix', ('nixos', 'nix'), ('*.nix',), ('text/x-nix',)),
+    'NuSMVLexer': ('pygments.lexers.smv', 'NuSMV', ('nusmv',), ('*.smv',), ()),
     'NumPyLexer': ('pygments.lexers.python', 'NumPy', ('numpy',), (), ()),
     'ObjdumpLexer': ('pygments.lexers.asm', 'objdump', ('objdump',), ('*.objdump',), ('text/x-objdump',)),
     'ObjectiveCLexer': ('pygments.lexers.objective', 'Objective-C', ('objective-c', 'objectivec', 'obj-c', 'objc'), ('*.m', '*.h'), ('text/x-objective-c',)),
@@ -256,25 +301,32 @@ LEXERS = {
     'ObjectiveJLexer': ('pygments.lexers.javascript', 'Objective-J', ('objective-j', 'objectivej', 'obj-j', 'objj'), ('*.j',), ('text/x-objective-j',)),
     'OcamlLexer': ('pygments.lexers.ml', 'OCaml', ('ocaml',), ('*.ml', '*.mli', '*.mll', '*.mly'), ('text/x-ocaml',)),
     'OctaveLexer': ('pygments.lexers.matlab', 'Octave', ('octave',), ('*.m',), ('text/octave',)),
+    'OdinLexer': ('pygments.lexers.archetype', 'ODIN', ('odin',), ('*.odin',), ('text/odin',)),
     'OocLexer': ('pygments.lexers.ooc', 'Ooc', ('ooc',), ('*.ooc',), ('text/x-ooc',)),
     'OpaLexer': ('pygments.lexers.ml', 'Opa', ('opa',), ('*.opa',), ('text/x-opa',)),
     'OpenEdgeLexer': ('pygments.lexers.business', 'OpenEdge ABL', ('openedge', 'abl', 'progress'), ('*.p', '*.cls'), ('text/x-openedge', 'application/x-openedge')),
+    'PacmanConfLexer': ('pygments.lexers.configs', 'PacmanConf', ('pacmanconf',), ('pacman.conf',), ()),
     'PanLexer': ('pygments.lexers.dsls', 'Pan', ('pan',), ('*.pan',), ()),
+    'ParaSailLexer': ('pygments.lexers.parasail', 'ParaSail', ('parasail',), ('*.psi', '*.psl'), ('text/x-parasail',)),
     'PawnLexer': ('pygments.lexers.pawn', 'Pawn', ('pawn',), ('*.p', '*.pwn', '*.inc'), ('text/x-pawn',)),
     'Perl6Lexer': ('pygments.lexers.perl', 'Perl6', ('perl6', 'pl6'), ('*.pl', '*.pm', '*.nqp', '*.p6', '*.6pl', '*.p6l', '*.pl6', '*.6pm', '*.p6m', '*.pm6', '*.t'), ('text/x-perl6', 'application/x-perl6')),
     'PerlLexer': ('pygments.lexers.perl', 'Perl', ('perl', 'pl'), ('*.pl', '*.pm', '*.t'), ('text/x-perl', 'application/x-perl')),
     'PhpLexer': ('pygments.lexers.php', 'PHP', ('php', 'php3', 'php4', 'php5'), ('*.php', '*.php[345]', '*.inc'), ('text/x-php',)),
     'PigLexer': ('pygments.lexers.jvm', 'Pig', ('pig',), ('*.pig',), ('text/x-pig',)),
     'PikeLexer': ('pygments.lexers.c_like', 'Pike', ('pike',), ('*.pike', '*.pmod'), ('text/x-pike',)),
+    'PkgConfigLexer': ('pygments.lexers.configs', 'PkgConfig', ('pkgconfig',), ('*.pc',), ()),
     'PlPgsqlLexer': ('pygments.lexers.sql', 'PL/pgSQL', ('plpgsql',), (), ('text/x-plpgsql',)),
     'PostScriptLexer': ('pygments.lexers.graphics', 'PostScript', ('postscript', 'postscr'), ('*.ps', '*.eps'), ('application/postscript',)),
     'PostgresConsoleLexer': ('pygments.lexers.sql', 'PostgreSQL console (psql)', ('psql', 'postgresql-console', 'postgres-console'), (), ('text/x-postgresql-psql',)),
     'PostgresLexer': ('pygments.lexers.sql', 'PostgreSQL SQL dialect', ('postgresql', 'postgres'), (), ('text/x-postgresql',)),
     'PovrayLexer': ('pygments.lexers.graphics', 'POVRay', ('pov',), ('*.pov', '*.inc'), ('text/x-povray',)),
     'PowerShellLexer': ('pygments.lexers.shell', 'PowerShell', ('powershell', 'posh', 'ps1', 'psm1'), ('*.ps1', '*.psm1'), ('text/x-powershell',)),
+    'PowerShellSessionLexer': ('pygments.lexers.shell', 'PowerShell Session', ('ps1con',), (), ()),
+    'PraatLexer': ('pygments.lexers.praat', 'Praat', ('praat',), ('*.praat', '*.proc', '*.psc'), ()),
     'PrologLexer': ('pygments.lexers.prolog', 'Prolog', ('prolog',), ('*.ecl', '*.prolog', '*.pro', '*.pl'), ('text/x-prolog',)),
     'PropertiesLexer': ('pygments.lexers.configs', 'Properties', ('properties', 'jproperties'), ('*.properties',), ('text/x-java-properties',)),
     'ProtoBufLexer': ('pygments.lexers.dsls', 'Protocol Buffer', ('protobuf', 'proto'), ('*.proto',), ()),
+    'PugLexer': ('pygments.lexers.html', 'Pug', ('pug', 'jade'), ('*.pug', '*.jade'), ('text/x-pug', 'text/x-jade')),
     'PuppetLexer': ('pygments.lexers.dsls', 'Puppet', ('puppet',), ('*.pp',), ()),
     'PyPyLogLexer': ('pygments.lexers.console', 'PyPy Log', ('pypylog', 'pypy'), ('*.pypylog',), ('application/x-pypylog',)),
     'Python3Lexer': ('pygments.lexers.python', 'Python 3', ('python3', 'py3'), (), ('text/x-python3', 'application/x-python3')),
@@ -283,8 +335,10 @@ LEXERS = {
     'PythonLexer': ('pygments.lexers.python', 'Python', ('python', 'py', 'sage'), ('*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript', '*.tac', '*.sage'), ('text/x-python', 'application/x-python')),
     'PythonTracebackLexer': ('pygments.lexers.python', 'Python Traceback', ('pytb',), ('*.pytb',), ('text/x-python-traceback',)),
     'QBasicLexer': ('pygments.lexers.basic', 'QBasic', ('qbasic', 'basic'), ('*.BAS', '*.bas'), ('text/basic',)),
-    'QmlLexer': ('pygments.lexers.webmisc', 'QML', ('qml',), ('*.qml',), ('application/x-qml',)),
+    'QVToLexer': ('pygments.lexers.qvt', 'QVTO', ('qvto', 'qvt'), ('*.qvto',), ()),
+    'QmlLexer': ('pygments.lexers.webmisc', 'QML', ('qml', 'qbs'), ('*.qml', '*.qbs'), ('application/x-qml', 'application/x-qt.qbs+qml')),
     'RConsoleLexer': ('pygments.lexers.r', 'RConsole', ('rconsole', 'rout'), ('*.Rout',), ()),
+    'RNCCompactLexer': ('pygments.lexers.rnc', 'Relax-NG Compact', ('rnc', 'rng-compact'), ('*.rnc',), ()),
     'RPMSpecLexer': ('pygments.lexers.installers', 'RPMSpec', ('spec',), ('*.spec',), ('text/x-rpm-spec',)),
     'RacketLexer': ('pygments.lexers.lisp', 'Racket', ('racket', 'rkt'), ('*.rkt', '*.rktd', '*.rktl'), ('text/x-racket', 'application/x-racket')),
     'RagelCLexer': ('pygments.lexers.parsers', 'Ragel in C Host', ('ragel-c',), ('*.rl',), ()),
@@ -304,13 +358,17 @@ LEXERS = {
     'ResourceLexer': ('pygments.lexers.resource', 'ResourceBundle', ('resource', 'resourcebundle'), ('*.txt',), ()),
     'RexxLexer': ('pygments.lexers.scripting', 'Rexx', ('rexx', 'arexx'), ('*.rexx', '*.rex', '*.rx', '*.arexx'), ('text/x-rexx',)),
     'RhtmlLexer': ('pygments.lexers.templates', 'RHTML', ('rhtml', 'html+erb', 'html+ruby'), ('*.rhtml',), ('text/html+ruby',)),
+    'RoboconfGraphLexer': ('pygments.lexers.roboconf', 'Roboconf Graph', ('roboconf-graph',), ('*.graph',), ()),
+    'RoboconfInstancesLexer': ('pygments.lexers.roboconf', 'Roboconf Instances', ('roboconf-instances',), ('*.instances',), ()),
     'RobotFrameworkLexer': ('pygments.lexers.robotframework', 'RobotFramework', ('robotframework',), ('*.txt', '*.robot'), ('text/x-robotframework',)),
     'RqlLexer': ('pygments.lexers.sql', 'RQL', ('rql',), ('*.rql',), ('text/x-rql',)),
     'RslLexer': ('pygments.lexers.dsls', 'RSL', ('rsl',), ('*.rsl',), ('text/rsl',)),
     'RstLexer': ('pygments.lexers.markup', 'reStructuredText', ('rst', 'rest', 'restructuredtext'), ('*.rst', '*.rest'), ('text/x-rst', 'text/prs.fallenstein.rst')),
+    'RtsLexer': ('pygments.lexers.trafficscript', 'TrafficScript', ('rts', 'trafficscript'), ('*.rts',), ()),
     'RubyConsoleLexer': ('pygments.lexers.ruby', 'Ruby irb session', ('rbcon', 'irb'), (), ('text/x-ruby-shellsession',)),
-    'RubyLexer': ('pygments.lexers.ruby', 'Ruby', ('rb', 'ruby', 'duby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby'), ('text/x-ruby', 'application/x-ruby')),
-    'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust',), ('*.rs',), ('text/x-rustsrc',)),
+    'RubyLexer': ('pygments.lexers.ruby', 'Ruby', ('rb', 'ruby', 'duby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby', 'Gemfile'), ('text/x-ruby', 'application/x-ruby')),
+    'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust',), ('*.rs', '*.rs.in'), ('text/rust',)),
+    'SASLexer': ('pygments.lexers.sas', 'SAS', ('sas',), ('*.SAS', '*.sas'), ('text/x-sas', 'text/sas', 'application/x-sas')),
     'SLexer': ('pygments.lexers.r', 'S', ('splus', 's', 'r'), ('*.S', '*.R', '.Rhistory', '.Rprofile', '.Renviron'), ('text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile')),
     'SMLLexer': ('pygments.lexers.ml', 'Standard ML', ('sml',), ('*.sml', '*.sig', '*.fun'), ('text/x-standardml', 'application/x-standardml')),
     'SassLexer': ('pygments.lexers.css', 'Sass', ('sass',), ('*.sass',), ('text/x-sass',)),
@@ -319,12 +377,14 @@ LEXERS = {
     'SchemeLexer': ('pygments.lexers.lisp', 'Scheme', ('scheme', 'scm'), ('*.scm', '*.ss'), ('text/x-scheme', 'application/x-scheme')),
     'ScilabLexer': ('pygments.lexers.matlab', 'Scilab', ('scilab',), ('*.sci', '*.sce', '*.tst'), ('text/scilab',)),
     'ScssLexer': ('pygments.lexers.css', 'SCSS', ('scss',), ('*.scss',), ('text/x-scss',)),
-    'ShellSessionLexer': ('pygments.lexers.shell', 'Shell Session', ('shell-session',), ('*.shell-session',), ('application/x-sh-session',)),
+    'ShenLexer': ('pygments.lexers.lisp', 'Shen', ('shen',), ('*.shen',), ('text/x-shen', 'application/x-shen')),
+    'SilverLexer': ('pygments.lexers.verification', 'Silver', ('silver',), ('*.sil', '*.vpr'), ()),
     'SlimLexer': ('pygments.lexers.webmisc', 'Slim', ('slim',), ('*.slim',), ('text/x-slim',)),
     'SmaliLexer': ('pygments.lexers.dalvik', 'Smali', ('smali',), ('*.smali',), ('text/smali',)),
     'SmalltalkLexer': ('pygments.lexers.smalltalk', 'Smalltalk', ('smalltalk', 'squeak', 'st'), ('*.st',), ('text/x-smalltalk',)),
     'SmartyLexer': ('pygments.lexers.templates', 'Smarty', ('smarty',), ('*.tpl',), ('application/x-smarty',)),
     'SnobolLexer': ('pygments.lexers.snobol', 'Snobol', ('snobol',), ('*.snobol',), ('text/x-snobol',)),
+    'SnowballLexer': ('pygments.lexers.dsls', 'Snowball', ('snowball',), ('*.sbl',), ()),
     'SourcePawnLexer': ('pygments.lexers.pawn', 'SourcePawn', ('sp',), ('*.sp',), ('text/x-sourcepawn',)),
     'SourcesListLexer': ('pygments.lexers.installers', 'Debian Sourcelist', ('sourceslist', 'sources.list', 'debsources'), ('sources.list',), ()),
     'SparqlLexer': ('pygments.lexers.rdf', 'SPARQL', ('sparql',), ('*.rq', '*.sparql'), ('application/sparql-query',)),
@@ -333,21 +393,37 @@ LEXERS = {
     'SquidConfLexer': ('pygments.lexers.configs', 'SquidConf', ('squidconf', 'squid.conf', 'squid'), ('squid.conf',), ('text/x-squidconf',)),
     'SspLexer': ('pygments.lexers.templates', 'Scalate Server Page', ('ssp',), ('*.ssp',), ('application/x-ssp',)),
     'StanLexer': ('pygments.lexers.modeling', 'Stan', ('stan',), ('*.stan',), ()),
+    'StataLexer': ('pygments.lexers.stata', 'Stata', ('stata', 'do'), ('*.do', '*.ado'), ('text/x-stata', 'text/stata', 'application/x-stata')),
+    'SuperColliderLexer': ('pygments.lexers.supercollider', 'SuperCollider', ('sc', 'supercollider'), ('*.sc', '*.scd'), ('application/supercollider', 'text/supercollider')),
     'SwiftLexer': ('pygments.lexers.objective', 'Swift', ('swift',), ('*.swift',), ('text/x-swift',)),
     'SwigLexer': ('pygments.lexers.c_like', 'SWIG', ('swig',), ('*.swg', '*.i'), ('text/swig',)),
     'SystemVerilogLexer': ('pygments.lexers.hdl', 'systemverilog', ('systemverilog', 'sv'), ('*.sv', '*.svh'), ('text/x-systemverilog',)),
+    'TAPLexer': ('pygments.lexers.testing', 'TAP', ('tap',), ('*.tap',), ()),
     'Tads3Lexer': ('pygments.lexers.int_fiction', 'TADS 3', ('tads3',), ('*.t',), ()),
+    'TasmLexer': ('pygments.lexers.asm', 'TASM', ('tasm',), ('*.asm', '*.ASM', '*.tasm'), ('text/x-tasm',)),
     'TclLexer': ('pygments.lexers.tcl', 'Tcl', ('tcl',), ('*.tcl', '*.rvt'), ('text/x-tcl', 'text/x-script.tcl', 'application/x-tcl')),
     'TcshLexer': ('pygments.lexers.shell', 'Tcsh', ('tcsh', 'csh'), ('*.tcsh', '*.csh'), ('application/x-csh',)),
+    'TcshSessionLexer': ('pygments.lexers.shell', 'Tcsh Session', ('tcshcon',), (), ()),
     'TeaTemplateLexer': ('pygments.lexers.templates', 'Tea', ('tea',), ('*.tea',), ('text/x-tea',)),
+    'TermcapLexer': ('pygments.lexers.configs', 'Termcap', ('termcap',), ('termcap', 'termcap.src'), ()),
+    'TerminfoLexer': ('pygments.lexers.configs', 'Terminfo', ('terminfo',), ('terminfo', 'terminfo.src'), ()),
+    'TerraformLexer': ('pygments.lexers.configs', 'Terraform', ('terraform', 'tf'), ('*.tf',), ('application/x-tf', 'application/x-terraform')),
     'TexLexer': ('pygments.lexers.markup', 'TeX', ('tex', 'latex'), ('*.tex', '*.aux', '*.toc'), ('text/x-tex', 'text/x-latex')),
     'TextLexer': ('pygments.lexers.special', 'Text only', ('text',), ('*.txt',), ('text/plain',)),
+    'ThriftLexer': ('pygments.lexers.dsls', 'Thrift', ('thrift',), ('*.thrift',), ('application/x-thrift',)),
     'TodotxtLexer': ('pygments.lexers.textfmts', 'Todotxt', ('todotxt',), ('todo.txt', '*.todotxt'), ('text/x-todo',)),
+    'TransactSqlLexer': ('pygments.lexers.sql', 'Transact-SQL', ('tsql', 't-sql'), ('*.sql',), ('text/x-tsql',)),
     'TreetopLexer': ('pygments.lexers.parsers', 'Treetop', ('treetop',), ('*.treetop', '*.tt'), ()),
+    'TurtleLexer': ('pygments.lexers.rdf', 'Turtle', ('turtle',), ('*.ttl',), ('text/turtle', 'application/x-turtle')),
     'TwigHtmlLexer': ('pygments.lexers.templates', 'HTML+Twig', ('html+twig',), ('*.twig',), ('text/html+twig',)),
     'TwigLexer': ('pygments.lexers.templates', 'Twig', ('twig',), (), ('application/x-twig',)),
-    'TypeScriptLexer': ('pygments.lexers.javascript', 'TypeScript', ('ts',), ('*.ts',), ('text/x-typescript',)),
+    'TypeScriptLexer': ('pygments.lexers.javascript', 'TypeScript', ('ts', 'typescript'), ('*.ts', '*.tsx'), ('text/x-typescript',)),
+    'TypoScriptCssDataLexer': ('pygments.lexers.typoscript', 'TypoScriptCssData', ('typoscriptcssdata',), (), ()),
+    'TypoScriptHtmlDataLexer': ('pygments.lexers.typoscript', 'TypoScriptHtmlData', ('typoscripthtmldata',), (), ()),
+    'TypoScriptLexer': ('pygments.lexers.typoscript', 'TypoScript', ('typoscript',), ('*.ts', '*.txt'), ('text/x-typoscript',)),
     'UrbiscriptLexer': ('pygments.lexers.urbi', 'UrbiScript', ('urbiscript',), ('*.u',), ('application/x-urbiscript',)),
+    'VCLLexer': ('pygments.lexers.varnish', 'VCL', ('vcl',), ('*.vcl',), ('text/x-vclsrc',)),
+    'VCLSnippetLexer': ('pygments.lexers.varnish', 'VCLSnippets', ('vclsnippets', 'vclsnippet'), (), ('text/x-vclsnippet',)),
     'VCTreeStatusLexer': ('pygments.lexers.console', 'VCTreeStatus', ('vctreestatus',), (), ()),
     'VGLLexer': ('pygments.lexers.dsls', 'VGL', ('vgl',), ('*.rpf',), ()),
     'ValaLexer': ('pygments.lexers.c_like', 'Vala', ('vala', 'vapi'), ('*.vala', '*.vapi'), ('text/x-vala',)),
@@ -359,20 +435,25 @@ LEXERS = {
     'VerilogLexer': ('pygments.lexers.hdl', 'verilog', ('verilog', 'v'), ('*.v',), ('text/x-verilog',)),
     'VhdlLexer': ('pygments.lexers.hdl', 'vhdl', ('vhdl',), ('*.vhdl', '*.vhd'), ('text/x-vhdl',)),
     'VimLexer': ('pygments.lexers.textedit', 'VimL', ('vim',), ('*.vim', '.vimrc', '.exrc', '.gvimrc', '_vimrc', '_exrc', '_gvimrc', 'vimrc', 'gvimrc'), ('text/x-vim',)),
+    'WDiffLexer': ('pygments.lexers.diff', 'WDiff', ('wdiff',), ('*.wdiff',), ()),
+    'WhileyLexer': ('pygments.lexers.whiley', 'Whiley', ('whiley',), ('*.whiley',), ('text/x-whiley',)),
+    'X10Lexer': ('pygments.lexers.x10', 'X10', ('x10', 'xten'), ('*.x10',), ('text/x-x10',)),
     'XQueryLexer': ('pygments.lexers.webmisc', 'XQuery', ('xquery', 'xqy', 'xq', 'xql', 'xqm'), ('*.xqy', '*.xquery', '*.xq', '*.xql', '*.xqm'), ('text/xquery', 'application/xquery')),
     'XmlDjangoLexer': ('pygments.lexers.templates', 'XML+Django/Jinja', ('xml+django', 'xml+jinja'), (), ('application/xml+django', 'application/xml+jinja')),
     'XmlErbLexer': ('pygments.lexers.templates', 'XML+Ruby', ('xml+erb', 'xml+ruby'), (), ('application/xml+ruby',)),
     'XmlLexer': ('pygments.lexers.html', 'XML', ('xml',), ('*.xml', '*.xsl', '*.rss', '*.xslt', '*.xsd', '*.wsdl', '*.wsf'), ('text/xml', 'application/xml', 'image/svg+xml', 'application/rss+xml', 'application/atom+xml')),
     'XmlPhpLexer': ('pygments.lexers.templates', 'XML+PHP', ('xml+php',), (), ('application/xml+php',)),
     'XmlSmartyLexer': ('pygments.lexers.templates', 'XML+Smarty', ('xml+smarty',), (), ('application/xml+smarty',)),
+    'XorgLexer': ('pygments.lexers.xorg', 'Xorg', ('xorg.conf',), ('xorg.conf',), ()),
     'XsltLexer': ('pygments.lexers.html', 'XSLT', ('xslt',), ('*.xsl', '*.xslt', '*.xpl'), ('application/xsl+xml', 'application/xslt+xml')),
     'XtendLexer': ('pygments.lexers.jvm', 'Xtend', ('xtend',), ('*.xtend',), ('text/x-xtend',)),
+    'XtlangLexer': ('pygments.lexers.lisp', 'xtlang', ('extempore',), ('*.xtm',), ()),
     'YamlJinjaLexer': ('pygments.lexers.templates', 'YAML+Jinja', ('yaml+jinja', 'salt', 'sls'), ('*.sls',), ('text/x-yaml+jinja', 'text/x-sls')),
     'YamlLexer': ('pygments.lexers.data', 'YAML', ('yaml',), ('*.yaml', '*.yml'), ('text/x-yaml',)),
     'ZephirLexer': ('pygments.lexers.php', 'Zephir', ('zephir',), ('*.zep',), ()),
 }
 
-if __name__ == '__main__':
+if __name__ == '__main__':  # pragma: no cover
     import sys
     import os
 
@@ -401,6 +482,13 @@ if __name__ == '__main__':
     # extract useful sourcecode from this file
     with open(__file__) as fp:
         content = fp.read()
+        # replace crnl to nl for Windows.
+        #
+        # Note that, originally, contributers should keep nl of master
+        # repository, for example by using some kind of automatic
+        # management EOL, like `EolExtension
+        #  <https://www.mercurial-scm.org/wiki/EolExtension>`.
+        content = content.replace("\r\n", "\n")
     header = content[:content.find('LEXERS = {')]
     footer = content[content.find("if __name__ == '__main__':"):]
 

packages/wakatime/packages/pygments/lexers/_mql_builtins.py

@@ -5,7 +5,7 @@
 
     Builtins for the MqlLexer.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 types = (

packages/wakatime/packages/pygments/lexers/_openedge_builtins.py

@@ -5,7 +5,7 @@
 
     Builtin list for the OpenEdgeLexer.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/_php_builtins.py

@@ -12,7 +12,7 @@
              internet connection. don't run that at home, use
              a server ;-)
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -4672,7 +4672,8 @@ MODULES = {'.NET': ('dotnet_load',),
            'xdiff_string_patch',
            'xdiff_string_rabdiff')}
 
-if __name__ == '__main__':
+
+if __name__ == '__main__':  # pragma: no cover
     import glob
     import os
     import pprint

packages/wakatime/packages/pygments/lexers/_postgres_builtins.py

@@ -5,7 +5,7 @@
 
     Self-updating data files for PostgreSQL lexer.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -507,7 +507,8 @@ PLPGSQL_KEYWORDS = (
     'RETURN', 'REVERSE', 'SQLSTATE', 'WHILE',
 )
 
-if __name__ == '__main__':
+
+if __name__ == '__main__':  # pragma: no cover
     import re
     try:
         from urllib import urlopen

packages/wakatime/packages/pygments/lexers/_scilab_builtins.py

@@ -5,7 +5,7 @@
 
     Builtin list for the ScilabLexer.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -3051,7 +3051,8 @@ variables_kw = (
     'xcoslib',
 )
 
-if __name__ == '__main__':
+
+if __name__ == '__main__':  # pragma: no cover
     import subprocess
     from pygments.util import format_lines, duplicates_removed
 

packages/wakatime/packages/pygments/lexers/_sourcemod_builtins.py

@@ -8,7 +8,7 @@
 
     Do not edit the FUNCTIONS list by hand.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -1091,7 +1091,9 @@ FUNCTIONS = (
     'SDKCall',
     'GetPlayerResourceEntity',
 )
-if __name__ == '__main__':
+
+
+if __name__ == '__main__':  # pragma: no cover
     import re
     import sys
     try:

packages/wakatime/packages/pygments/lexers/_stan_builtins.py

@@ -4,9 +4,9 @@
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     This file contains the names of functions for Stan used by
-    ``pygments.lexers.math.StanLexer. This is for Stan language version 2.4.0.
+    ``pygments.lexers.math.StanLexer. This is for Stan language version 2.8.0.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -16,10 +16,12 @@ KEYWORDS = (
     'if',
     'in',
     'increment_log_prob',
+    'integrate_ode',
     'lp__',
     'print',
+    'reject',
     'return',
-    'while',
+    'while'
 )
 
 TYPES = (
@@ -33,11 +35,11 @@ TYPES = (
     'positive_ordered',
     'real',
     'row_vector',
+    'row_vectormatrix',
     'simplex',
     'unit_vector',
     'vector',
-    'void',
-)
+    'void')
 
 FUNCTIONS = (
     'Phi',
@@ -45,6 +47,8 @@ FUNCTIONS = (
     'abs',
     'acos',
     'acosh',
+    'append_col',
+    'append_row',
     'asin',
     'asinh',
     'atan',
@@ -100,6 +104,11 @@ FUNCTIONS = (
     'cos',
     'cosh',
     'crossprod',
+    'csr_extract_u',
+    'csr_extract_v',
+    'csr_extract_w',
+    'csr_matrix_times_vector',
+    'csr_to_dense_matrix',
     'cumulative_sum',
     'determinant',
     'diag_matrix',
@@ -144,6 +153,11 @@ FUNCTIONS = (
     'fmax',
     'fmin',
     'fmod',
+    'frechet_ccdf_log',
+    'frechet_cdf',
+    'frechet_cdf_log',
+    'frechet_log',
+    'frechet_rng',
     'gamma_ccdf_log',
     'gamma_cdf',
     'gamma_cdf_log',
@@ -152,6 +166,7 @@ FUNCTIONS = (
     'gamma_q',
     'gamma_rng',
     'gaussian_dlm_obs_log',
+    'get_lp',
     'gumbel_ccdf_log',
     'gumbel_cdf',
     'gumbel_cdf_log',
@@ -176,19 +191,21 @@ FUNCTIONS = (
     'inv_gamma_log',
     'inv_gamma_rng',
     'inv_logit',
+    'inv_phi',
     'inv_sqrt',
     'inv_square',
     'inv_wishart_log',
     'inv_wishart_rng',
     'inverse',
     'inverse_spd',
+    'is_inf',
+    'is_nan',
     'lbeta',
     'lgamma',
     'lkj_corr_cholesky_log',
     'lkj_corr_cholesky_rng',
     'lkj_corr_log',
     'lkj_corr_rng',
-    'lkj_cov_log',
     'lmgamma',
     'log',
     'log10',
@@ -202,6 +219,7 @@ FUNCTIONS = (
     'log_diff_exp',
     'log_falling_factorial',
     'log_inv_logit',
+    'log_mix',
     'log_rising_factorial',
     'log_softmax',
     'log_sum_exp',
@@ -224,6 +242,7 @@ FUNCTIONS = (
     'min',
     'modified_bessel_first_kind',
     'modified_bessel_second_kind',
+    'multi_gp_cholesky_log',
     'multi_gp_log',
     'multi_normal_cholesky_log',
     'multi_normal_cholesky_rng',
@@ -236,6 +255,9 @@ FUNCTIONS = (
     'multinomial_rng',
     'multiply_log',
     'multiply_lower_tri_self_transpose',
+    'neg_binomial_2_ccdf_log',
+    'neg_binomial_2_cdf',
+    'neg_binomial_2_cdf_log',
     'neg_binomial_2_log',
     'neg_binomial_2_log_log',
     'neg_binomial_2_log_rng',
@@ -252,6 +274,7 @@ FUNCTIONS = (
     'normal_log',
     'normal_rng',
     'not_a_number',
+    'num_elements',
     'ordered_logistic_log',
     'ordered_logistic_rng',
     'owens_t',
@@ -260,12 +283,18 @@ FUNCTIONS = (
     'pareto_cdf_log',
     'pareto_log',
     'pareto_rng',
+    'pareto_type_2_ccdf_log',
+    'pareto_type_2_cdf',
+    'pareto_type_2_cdf_log',
+    'pareto_type_2_log',
+    'pareto_type_2_rng',
     'pi',
     'poisson_ccdf_log',
     'poisson_cdf',
     'poisson_cdf_log',
     'poisson_log',
     'poisson_log_log',
+    'poisson_log_rng',
     'poisson_rng',
     'positive_infinity',
     'pow',
@@ -353,8 +382,9 @@ FUNCTIONS = (
     'weibull_cdf_log',
     'weibull_log',
     'weibull_rng',
+    'wiener_log',
     'wishart_log',
-    'wishart_rng',
+    'wishart_rng'
 )
 
 DISTRIBUTIONS = (
@@ -372,6 +402,7 @@ DISTRIBUTIONS = (
     'double_exponential',
     'exp_mod_normal',
     'exponential',
+    'frechet',
     'gamma',
     'gaussian_dlm_obs',
     'gumbel',
@@ -381,10 +412,10 @@ DISTRIBUTIONS = (
     'inv_wishart',
     'lkj_corr',
     'lkj_corr_cholesky',
-    'lkj_cov',
     'logistic',
     'lognormal',
     'multi_gp',
+    'multi_gp_cholesky',
     'multi_normal',
     'multi_normal_cholesky',
     'multi_normal_prec',
@@ -396,6 +427,7 @@ DISTRIBUTIONS = (
     'normal',
     'ordered_logistic',
     'pareto',
+    'pareto_type_2',
     'poisson',
     'poisson_log',
     'rayleigh',
@@ -405,7 +437,8 @@ DISTRIBUTIONS = (
     'uniform',
     'von_mises',
     'weibull',
-    'wishart',
+    'wiener',
+    'wishart'
 )
 
 RESERVED = (
@@ -494,5 +527,6 @@ RESERVED = (
     'volatile',
     'wchar_t',
     'xor',
-    'xor_eq',
+    'xor_eq'
 )
+

packages/wakatime/packages/pygments/lexers/_stata_builtins.py

@@ -0,0 +1,419 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers._stata_builtins
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Builtins for Stata
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+
+builtins_base = (
+    "if", "else", "in", "foreach", "for", "forv", "forva",
+    "forval", "forvalu", "forvalue", "forvalues", "by", "bys",
+    "bysort", "quietly", "qui", "about", "ac",
+    "ac_7", "acprplot", "acprplot_7", "adjust", "ado", "adopath",
+    "adoupdate", "alpha", "ameans", "an", "ano", "anov", "anova",
+    "anova_estat", "anova_terms", "anovadef", "aorder", "ap", "app",
+    "appe", "appen", "append", "arch", "arch_dr", "arch_estat",
+    "arch_p", "archlm", "areg", "areg_p", "args", "arima",
+    "arima_dr", "arima_estat", "arima_p", "as", "asmprobit",
+    "asmprobit_estat", "asmprobit_lf", "asmprobit_mfx__dlg",
+    "asmprobit_p", "ass", "asse", "asser", "assert", "avplot",
+    "avplot_7", "avplots", "avplots_7", "bcskew0", "bgodfrey",
+    "binreg", "bip0_lf", "biplot", "bipp_lf", "bipr_lf",
+    "bipr_p", "biprobit", "bitest", "bitesti", "bitowt", "blogit",
+    "bmemsize", "boot", "bootsamp", "bootstrap", "bootstrap_8",
+    "boxco_l", "boxco_p", "boxcox", "boxcox_6", "boxcox_p",
+    "bprobit", "br", "break", "brier", "bro", "brow", "brows",
+    "browse", "brr", "brrstat", "bs", "bs_7", "bsampl_w",
+    "bsample", "bsample_7", "bsqreg", "bstat", "bstat_7", "bstat_8",
+    "bstrap", "bstrap_7", "ca", "ca_estat", "ca_p", "cabiplot",
+    "camat", "canon", "canon_8", "canon_8_p", "canon_estat",
+    "canon_p", "cap", "caprojection", "capt", "captu", "captur",
+    "capture", "cat", "cc", "cchart", "cchart_7", "cci",
+    "cd", "censobs_table", "centile", "cf", "char", "chdir",
+    "checkdlgfiles", "checkestimationsample", "checkhlpfiles",
+    "checksum", "chelp", "ci", "cii", "cl", "class", "classutil",
+    "clear", "cli", "clis", "clist", "clo", "clog", "clog_lf",
+    "clog_p", "clogi", "clogi_sw", "clogit", "clogit_lf",
+    "clogit_p", "clogitp", "clogl_sw", "cloglog", "clonevar",
+    "clslistarray", "cluster", "cluster_measures", "cluster_stop",
+    "cluster_tree", "cluster_tree_8", "clustermat", "cmdlog",
+    "cnr", "cnre", "cnreg", "cnreg_p", "cnreg_sw", "cnsreg",
+    "codebook", "collaps4", "collapse", "colormult_nb",
+    "colormult_nw", "compare", "compress", "conf", "confi",
+    "confir", "confirm", "conren", "cons", "const", "constr",
+    "constra", "constrai", "constrain", "constraint", "continue",
+    "contract", "copy", "copyright", "copysource", "cor", "corc",
+    "corr", "corr2data", "corr_anti", "corr_kmo", "corr_smc",
+    "corre", "correl", "correla", "correlat", "correlate",
+    "corrgram", "cou", "coun", "count", "cox", "cox_p", "cox_sw",
+    "coxbase", "coxhaz", "coxvar", "cprplot", "cprplot_7",
+    "crc", "cret", "cretu", "cretur", "creturn", "cross", "cs",
+    "cscript", "cscript_log", "csi", "ct", "ct_is", "ctset",
+    "ctst_5", "ctst_st", "cttost", "cumsp", "cumsp_7", "cumul",
+    "cusum", "cusum_7", "cutil", "d", "datasig", "datasign",
+    "datasigna", "datasignat", "datasignatu", "datasignatur",
+    "datasignature", "datetof", "db", "dbeta", "de", "dec",
+    "deco", "decod", "decode", "deff", "des", "desc", "descr",
+    "descri", "describ", "describe", "destring", "dfbeta",
+    "dfgls", "dfuller", "di", "di_g", "dir", "dirstats", "dis",
+    "discard", "disp", "disp_res", "disp_s", "displ", "displa",
+    "display", "distinct", "do", "doe", "doed", "doedi",
+    "doedit", "dotplot", "dotplot_7", "dprobit", "drawnorm",
+    "drop", "ds", "ds_util", "dstdize", "duplicates", "durbina",
+    "dwstat", "dydx", "e", "ed", "edi", "edit", "egen",
+    "eivreg", "emdef", "en", "enc", "enco", "encod", "encode",
+    "eq", "erase", "ereg", "ereg_lf", "ereg_p", "ereg_sw",
+    "ereghet", "ereghet_glf", "ereghet_glf_sh", "ereghet_gp",
+    "ereghet_ilf", "ereghet_ilf_sh", "ereghet_ip", "eret",
+    "eretu", "eretur", "ereturn", "err", "erro", "error", "est",
+    "est_cfexist", "est_cfname", "est_clickable", "est_expand",
+    "est_hold", "est_table", "est_unhold", "est_unholdok",
+    "estat", "estat_default", "estat_summ", "estat_vce_only",
+    "esti", "estimates", "etodow", "etof", "etomdy", "ex",
+    "exi", "exit", "expand", "expandcl", "fac", "fact", "facto",
+    "factor", "factor_estat", "factor_p", "factor_pca_rotated",
+    "factor_rotate", "factormat", "fcast", "fcast_compute",
+    "fcast_graph", "fdades", "fdadesc", "fdadescr", "fdadescri",
+    "fdadescrib", "fdadescribe", "fdasav", "fdasave", "fdause",
+    "fh_st", "open", "read", "close",
+    "file", "filefilter", "fillin", "find_hlp_file", "findfile",
+    "findit", "findit_7", "fit", "fl", "fli", "flis", "flist",
+    "for5_0", "form", "forma", "format", "fpredict", "frac_154",
+    "frac_adj", "frac_chk", "frac_cox", "frac_ddp", "frac_dis",
+    "frac_dv", "frac_in", "frac_mun", "frac_pp", "frac_pq",
+    "frac_pv", "frac_wgt", "frac_xo", "fracgen", "fracplot",
+    "fracplot_7", "fracpoly", "fracpred", "fron_ex", "fron_hn",
+    "fron_p", "fron_tn", "fron_tn2", "frontier", "ftodate", "ftoe",
+    "ftomdy", "ftowdate", "g", "gamhet_glf", "gamhet_gp",
+    "gamhet_ilf", "gamhet_ip", "gamma", "gamma_d2", "gamma_p",
+    "gamma_sw", "gammahet", "gdi_hexagon", "gdi_spokes", "ge",
+    "gen", "gene", "gener", "genera", "generat", "generate",
+    "genrank", "genstd", "genvmean", "gettoken", "gl", "gladder",
+    "gladder_7", "glim_l01", "glim_l02", "glim_l03", "glim_l04",
+    "glim_l05", "glim_l06", "glim_l07", "glim_l08", "glim_l09",
+    "glim_l10", "glim_l11", "glim_l12", "glim_lf", "glim_mu",
+    "glim_nw1", "glim_nw2", "glim_nw3", "glim_p", "glim_v1",
+    "glim_v2", "glim_v3", "glim_v4", "glim_v5", "glim_v6",
+    "glim_v7", "glm", "glm_6", "glm_p", "glm_sw", "glmpred", "glo",
+    "glob", "globa", "global", "glogit", "glogit_8", "glogit_p",
+    "gmeans", "gnbre_lf", "gnbreg", "gnbreg_5", "gnbreg_p",
+    "gomp_lf", "gompe_sw", "gomper_p", "gompertz", "gompertzhet",
+    "gomphet_glf", "gomphet_glf_sh", "gomphet_gp", "gomphet_ilf",
+    "gomphet_ilf_sh", "gomphet_ip", "gphdot", "gphpen",
+    "gphprint", "gprefs", "gprobi_p", "gprobit", "gprobit_8", "gr",
+    "gr7", "gr_copy", "gr_current", "gr_db", "gr_describe",
+    "gr_dir", "gr_draw", "gr_draw_replay", "gr_drop", "gr_edit",
+    "gr_editviewopts", "gr_example", "gr_example2", "gr_export",
+    "gr_print", "gr_qscheme", "gr_query", "gr_read", "gr_rename",
+    "gr_replay", "gr_save", "gr_set", "gr_setscheme", "gr_table",
+    "gr_undo", "gr_use", "graph", "graph7", "grebar", "greigen",
+    "greigen_7", "greigen_8", "grmeanby", "grmeanby_7",
+    "gs_fileinfo", "gs_filetype", "gs_graphinfo", "gs_stat",
+    "gsort", "gwood", "h", "hadimvo", "hareg", "hausman",
+    "haver", "he", "heck_d2", "heckma_p", "heckman", "heckp_lf",
+    "heckpr_p", "heckprob", "hel", "help", "hereg", "hetpr_lf",
+    "hetpr_p", "hetprob", "hettest", "hexdump", "hilite",
+    "hist", "hist_7", "histogram", "hlogit", "hlu", "hmeans",
+    "hotel", "hotelling", "hprobit", "hreg", "hsearch", "icd9",
+    "icd9_ff", "icd9p", "iis", "impute", "imtest", "inbase",
+    "include", "inf", "infi", "infil", "infile", "infix", "inp",
+    "inpu", "input", "ins", "insheet", "insp", "inspe",
+    "inspec", "inspect", "integ", "inten", "intreg", "intreg_7",
+    "intreg_p", "intrg2_ll", "intrg_ll", "intrg_ll2", "ipolate",
+    "iqreg", "ir", "irf", "irf_create", "irfm", "iri", "is_svy",
+    "is_svysum", "isid", "istdize", "ivprob_1_lf", "ivprob_lf",
+    "ivprobit", "ivprobit_p", "ivreg", "ivreg_footnote",
+    "ivtob_1_lf", "ivtob_lf", "ivtobit", "ivtobit_p", "jackknife",
+    "jacknife", "jknife", "jknife_6", "jknife_8", "jkstat",
+    "joinby", "kalarma1", "kap", "kap_3", "kapmeier", "kappa",
+    "kapwgt", "kdensity", "kdensity_7", "keep", "ksm", "ksmirnov",
+    "ktau", "kwallis", "l", "la", "lab", "labe", "label",
+    "labelbook", "ladder", "levels", "levelsof", "leverage",
+    "lfit", "lfit_p", "li", "lincom", "line", "linktest",
+    "lis", "list", "lloghet_glf", "lloghet_glf_sh", "lloghet_gp",
+    "lloghet_ilf", "lloghet_ilf_sh", "lloghet_ip", "llogi_sw",
+    "llogis_p", "llogist", "llogistic", "llogistichet",
+    "lnorm_lf", "lnorm_sw", "lnorma_p", "lnormal", "lnormalhet",
+    "lnormhet_glf", "lnormhet_glf_sh", "lnormhet_gp",
+    "lnormhet_ilf", "lnormhet_ilf_sh", "lnormhet_ip", "lnskew0",
+    "loadingplot", "loc", "loca", "local", "log", "logi",
+    "logis_lf", "logistic", "logistic_p", "logit", "logit_estat",
+    "logit_p", "loglogs", "logrank", "loneway", "lookfor",
+    "lookup", "lowess", "lowess_7", "lpredict", "lrecomp", "lroc",
+    "lroc_7", "lrtest", "ls", "lsens", "lsens_7", "lsens_x",
+    "lstat", "ltable", "ltable_7", "ltriang", "lv", "lvr2plot",
+    "lvr2plot_7", "m", "ma", "mac", "macr", "macro", "makecns",
+    "man", "manova", "manova_estat", "manova_p", "manovatest",
+    "mantel", "mark", "markin", "markout", "marksample", "mat",
+    "mat_capp", "mat_order", "mat_put_rr", "mat_rapp", "mata",
+    "mata_clear", "mata_describe", "mata_drop", "mata_matdescribe",
+    "mata_matsave", "mata_matuse", "mata_memory", "mata_mlib",
+    "mata_mosave", "mata_rename", "mata_which", "matalabel",
+    "matcproc", "matlist", "matname", "matr", "matri",
+    "matrix", "matrix_input__dlg", "matstrik", "mcc", "mcci",
+    "md0_", "md1_", "md1debug_", "md2_", "md2debug_", "mds",
+    "mds_estat", "mds_p", "mdsconfig", "mdslong", "mdsmat",
+    "mdsshepard", "mdytoe", "mdytof", "me_derd", "mean",
+    "means", "median", "memory", "memsize", "meqparse", "mer",
+    "merg", "merge", "mfp", "mfx", "mhelp", "mhodds", "minbound",
+    "mixed_ll", "mixed_ll_reparm", "mkassert", "mkdir",
+    "mkmat", "mkspline", "ml", "ml_5", "ml_adjs", "ml_bhhhs",
+    "ml_c_d", "ml_check", "ml_clear", "ml_cnt", "ml_debug",
+    "ml_defd", "ml_e0", "ml_e0_bfgs", "ml_e0_cycle", "ml_e0_dfp",
+    "ml_e0i", "ml_e1", "ml_e1_bfgs", "ml_e1_bhhh", "ml_e1_cycle",
+    "ml_e1_dfp", "ml_e2", "ml_e2_cycle", "ml_ebfg0", "ml_ebfr0",
+    "ml_ebfr1", "ml_ebh0q", "ml_ebhh0", "ml_ebhr0", "ml_ebr0i",
+    "ml_ecr0i", "ml_edfp0", "ml_edfr0", "ml_edfr1", "ml_edr0i",
+    "ml_eds", "ml_eer0i", "ml_egr0i", "ml_elf", "ml_elf_bfgs",
+    "ml_elf_bhhh", "ml_elf_cycle", "ml_elf_dfp", "ml_elfi",
+    "ml_elfs", "ml_enr0i", "ml_enrr0", "ml_erdu0", "ml_erdu0_bfgs",
+    "ml_erdu0_bhhh", "ml_erdu0_bhhhq", "ml_erdu0_cycle",
+    "ml_erdu0_dfp", "ml_erdu0_nrbfgs", "ml_exde", "ml_footnote",
+    "ml_geqnr", "ml_grad0", "ml_graph", "ml_hbhhh", "ml_hd0",
+    "ml_hold", "ml_init", "ml_inv", "ml_log", "ml_max",
+    "ml_mlout", "ml_mlout_8", "ml_model", "ml_nb0", "ml_opt",
+    "ml_p", "ml_plot", "ml_query", "ml_rdgrd", "ml_repor",
+    "ml_s_e", "ml_score", "ml_searc", "ml_technique", "ml_unhold",
+    "mleval", "mlf_", "mlmatbysum", "mlmatsum", "mlog", "mlogi",
+    "mlogit", "mlogit_footnote", "mlogit_p", "mlopts", "mlsum",
+    "mlvecsum", "mnl0_", "mor", "more", "mov", "move", "mprobit",
+    "mprobit_lf", "mprobit_p", "mrdu0_", "mrdu1_", "mvdecode",
+    "mvencode", "mvreg", "mvreg_estat", "n", "nbreg",
+    "nbreg_al", "nbreg_lf", "nbreg_p", "nbreg_sw", "nestreg", "net",
+    "newey", "newey_7", "newey_p", "news", "nl", "nl_7", "nl_9",
+    "nl_9_p", "nl_p", "nl_p_7", "nlcom", "nlcom_p", "nlexp2",
+    "nlexp2_7", "nlexp2a", "nlexp2a_7", "nlexp3", "nlexp3_7",
+    "nlgom3", "nlgom3_7", "nlgom4", "nlgom4_7", "nlinit", "nllog3",
+    "nllog3_7", "nllog4", "nllog4_7", "nlog_rd", "nlogit",
+    "nlogit_p", "nlogitgen", "nlogittree", "nlpred", "no",
+    "nobreak", "noi", "nois", "noisi", "noisil", "noisily", "note",
+    "notes", "notes_dlg", "nptrend", "numlabel", "numlist", "odbc",
+    "old_ver", "olo", "olog", "ologi", "ologi_sw", "ologit",
+    "ologit_p", "ologitp", "on", "one", "onew", "onewa", "oneway",
+    "op_colnm", "op_comp", "op_diff", "op_inv", "op_str", "opr",
+    "opro", "oprob", "oprob_sw", "oprobi", "oprobi_p", "oprobit",
+    "oprobitp", "opts_exclusive", "order", "orthog", "orthpoly",
+    "ou", "out", "outf", "outfi", "outfil", "outfile", "outs",
+    "outsh", "outshe", "outshee", "outsheet", "ovtest", "pac",
+    "pac_7", "palette", "parse", "parse_dissim", "pause", "pca",
+    "pca_8", "pca_display", "pca_estat", "pca_p", "pca_rotate",
+    "pcamat", "pchart", "pchart_7", "pchi", "pchi_7", "pcorr",
+    "pctile", "pentium", "pergram", "pergram_7", "permute",
+    "permute_8", "personal", "peto_st", "pkcollapse", "pkcross",
+    "pkequiv", "pkexamine", "pkexamine_7", "pkshape", "pksumm",
+    "pksumm_7", "pl", "plo", "plot", "plugin", "pnorm",
+    "pnorm_7", "poisgof", "poiss_lf", "poiss_sw", "poisso_p",
+    "poisson", "poisson_estat", "post", "postclose", "postfile",
+    "postutil", "pperron", "pr", "prais", "prais_e", "prais_e2",
+    "prais_p", "predict", "predictnl", "preserve", "print",
+    "pro", "prob", "probi", "probit", "probit_estat", "probit_p",
+    "proc_time", "procoverlay", "procrustes", "procrustes_estat",
+    "procrustes_p", "profiler", "prog", "progr", "progra",
+    "program", "prop", "proportion", "prtest", "prtesti", "pwcorr",
+    "pwd", "q", "s", "qby", "qbys", "qchi", "qchi_7", "qladder",
+    "qladder_7", "qnorm", "qnorm_7", "qqplot", "qqplot_7", "qreg",
+    "qreg_c", "qreg_p", "qreg_sw", "qu", "quadchk", "quantile",
+    "quantile_7", "que", "quer", "query", "range", "ranksum",
+    "ratio", "rchart", "rchart_7", "rcof", "recast", "reclink",
+    "recode", "reg", "reg3", "reg3_p", "regdw", "regr", "regre",
+    "regre_p2", "regres", "regres_p", "regress", "regress_estat",
+    "regriv_p", "remap", "ren", "rena", "renam", "rename",
+    "renpfix", "repeat", "replace", "report", "reshape",
+    "restore", "ret", "retu", "retur", "return", "rm", "rmdir",
+    "robvar", "roccomp", "roccomp_7", "roccomp_8", "rocf_lf",
+    "rocfit", "rocfit_8", "rocgold", "rocplot", "rocplot_7",
+    "roctab", "roctab_7", "rolling", "rologit", "rologit_p",
+    "rot", "rota", "rotat", "rotate", "rotatemat", "rreg",
+    "rreg_p", "ru", "run", "runtest", "rvfplot", "rvfplot_7",
+    "rvpplot", "rvpplot_7", "sa", "safesum", "sample",
+    "sampsi", "sav", "save", "savedresults", "saveold", "sc",
+    "sca", "scal", "scala", "scalar", "scatter", "scm_mine",
+    "sco", "scob_lf", "scob_p", "scobi_sw", "scobit", "scor",
+    "score", "scoreplot", "scoreplot_help", "scree", "screeplot",
+    "screeplot_help", "sdtest", "sdtesti", "se", "search",
+    "separate", "seperate", "serrbar", "serrbar_7", "serset", "set",
+    "set_defaults", "sfrancia", "sh", "she", "shel", "shell",
+    "shewhart", "shewhart_7", "signestimationsample", "signrank",
+    "signtest", "simul", "simul_7", "simulate", "simulate_8",
+    "sktest", "sleep", "slogit", "slogit_d2", "slogit_p", "smooth",
+    "snapspan", "so", "sor", "sort", "spearman", "spikeplot",
+    "spikeplot_7", "spikeplt", "spline_x", "split", "sqreg",
+    "sqreg_p", "sret", "sretu", "sretur", "sreturn", "ssc", "st",
+    "st_ct", "st_hc", "st_hcd", "st_hcd_sh", "st_is", "st_issys",
+    "st_note", "st_promo", "st_set", "st_show", "st_smpl",
+    "st_subid", "stack", "statsby", "statsby_8", "stbase", "stci",
+    "stci_7", "stcox", "stcox_estat", "stcox_fr", "stcox_fr_ll",
+    "stcox_p", "stcox_sw", "stcoxkm", "stcoxkm_7", "stcstat",
+    "stcurv", "stcurve", "stcurve_7", "stdes", "stem", "stepwise",
+    "stereg", "stfill", "stgen", "stir", "stjoin", "stmc", "stmh",
+    "stphplot", "stphplot_7", "stphtest", "stphtest_7",
+    "stptime", "strate", "strate_7", "streg", "streg_sw", "streset",
+    "sts", "sts_7", "stset", "stsplit", "stsum", "sttocc",
+    "sttoct", "stvary", "stweib", "su", "suest", "suest_8",
+    "sum", "summ", "summa", "summar", "summari", "summariz",
+    "summarize", "sunflower", "sureg", "survcurv", "survsum",
+    "svar", "svar_p", "svmat", "svy", "svy_disp", "svy_dreg",
+    "svy_est", "svy_est_7", "svy_estat", "svy_get", "svy_gnbreg_p",
+    "svy_head", "svy_header", "svy_heckman_p", "svy_heckprob_p",
+    "svy_intreg_p", "svy_ivreg_p", "svy_logistic_p", "svy_logit_p",
+    "svy_mlogit_p", "svy_nbreg_p", "svy_ologit_p", "svy_oprobit_p",
+    "svy_poisson_p", "svy_probit_p", "svy_regress_p", "svy_sub",
+    "svy_sub_7", "svy_x", "svy_x_7", "svy_x_p", "svydes",
+    "svydes_8", "svygen", "svygnbreg", "svyheckman", "svyheckprob",
+    "svyintreg", "svyintreg_7", "svyintrg", "svyivreg", "svylc",
+    "svylog_p", "svylogit", "svymarkout", "svymarkout_8",
+    "svymean", "svymlog", "svymlogit", "svynbreg", "svyolog",
+    "svyologit", "svyoprob", "svyoprobit", "svyopts",
+    "svypois", "svypois_7", "svypoisson", "svyprobit", "svyprobt",
+    "svyprop", "svyprop_7", "svyratio", "svyreg", "svyreg_p",
+    "svyregress", "svyset", "svyset_7", "svyset_8", "svytab",
+    "svytab_7", "svytest", "svytotal", "sw", "sw_8", "swcnreg",
+    "swcox", "swereg", "swilk", "swlogis", "swlogit",
+    "swologit", "swoprbt", "swpois", "swprobit", "swqreg",
+    "swtobit", "swweib", "symmetry", "symmi", "symplot",
+    "symplot_7", "syntax", "sysdescribe", "sysdir", "sysuse",
+    "szroeter", "ta", "tab", "tab1", "tab2", "tab_or", "tabd",
+    "tabdi", "tabdis", "tabdisp", "tabi", "table", "tabodds",
+    "tabodds_7", "tabstat", "tabu", "tabul", "tabula", "tabulat",
+    "tabulate", "te", "tempfile", "tempname", "tempvar", "tes",
+    "test", "testnl", "testparm", "teststd", "tetrachoric",
+    "time_it", "timer", "tis", "tob", "tobi", "tobit", "tobit_p",
+    "tobit_sw", "token", "tokeni", "tokeniz", "tokenize",
+    "tostring", "total", "translate", "translator", "transmap",
+    "treat_ll", "treatr_p", "treatreg", "trim", "trnb_cons",
+    "trnb_mean", "trpoiss_d2", "trunc_ll", "truncr_p", "truncreg",
+    "tsappend", "tset", "tsfill", "tsline", "tsline_ex",
+    "tsreport", "tsrevar", "tsrline", "tsset", "tssmooth",
+    "tsunab", "ttest", "ttesti", "tut_chk", "tut_wait", "tutorial",
+    "tw", "tware_st", "two", "twoway", "twoway__fpfit_serset",
+    "twoway__function_gen", "twoway__histogram_gen",
+    "twoway__ipoint_serset", "twoway__ipoints_serset",
+    "twoway__kdensity_gen", "twoway__lfit_serset",
+    "twoway__normgen_gen", "twoway__pci_serset",
+    "twoway__qfit_serset", "twoway__scatteri_serset",
+    "twoway__sunflower_gen", "twoway_ksm_serset", "ty", "typ",
+    "type", "typeof", "u", "unab", "unabbrev", "unabcmd",
+    "update", "us", "use", "uselabel", "var", "var_mkcompanion",
+    "var_p", "varbasic", "varfcast", "vargranger", "varirf",
+    "varirf_add", "varirf_cgraph", "varirf_create", "varirf_ctable",
+    "varirf_describe", "varirf_dir", "varirf_drop", "varirf_erase",
+    "varirf_graph", "varirf_ograph", "varirf_rename", "varirf_set",
+    "varirf_table", "varlist", "varlmar", "varnorm", "varsoc",
+    "varstable", "varstable_w", "varstable_w2", "varwle",
+    "vce", "vec", "vec_fevd", "vec_mkphi", "vec_p", "vec_p_w",
+    "vecirf_create", "veclmar", "veclmar_w", "vecnorm",
+    "vecnorm_w", "vecrank", "vecstable", "verinst", "vers",
+    "versi", "versio", "version", "view", "viewsource", "vif",
+    "vwls", "wdatetof", "webdescribe", "webseek", "webuse",
+    "weib1_lf", "weib2_lf", "weib_lf", "weib_lf0", "weibhet_glf",
+    "weibhet_glf_sh", "weibhet_glfa", "weibhet_glfa_sh",
+    "weibhet_gp", "weibhet_ilf", "weibhet_ilf_sh", "weibhet_ilfa",
+    "weibhet_ilfa_sh", "weibhet_ip", "weibu_sw", "weibul_p",
+    "weibull", "weibull_c", "weibull_s", "weibullhet",
+    "wh", "whelp", "whi", "which", "whil", "while", "wilc_st",
+    "wilcoxon", "win", "wind", "windo", "window", "winexec",
+    "wntestb", "wntestb_7", "wntestq", "xchart", "xchart_7",
+    "xcorr", "xcorr_7", "xi", "xi_6", "xmlsav", "xmlsave",
+    "xmluse", "xpose", "xsh", "xshe", "xshel", "xshell",
+    "xt_iis", "xt_tis", "xtab_p", "xtabond", "xtbin_p",
+    "xtclog", "xtcloglog", "xtcloglog_8", "xtcloglog_d2",
+    "xtcloglog_pa_p", "xtcloglog_re_p", "xtcnt_p", "xtcorr",
+    "xtdata", "xtdes", "xtfront_p", "xtfrontier", "xtgee",
+    "xtgee_elink", "xtgee_estat", "xtgee_makeivar", "xtgee_p",
+    "xtgee_plink", "xtgls", "xtgls_p", "xthaus", "xthausman",
+    "xtht_p", "xthtaylor", "xtile", "xtint_p", "xtintreg",
+    "xtintreg_8", "xtintreg_d2", "xtintreg_p", "xtivp_1",
+    "xtivp_2", "xtivreg", "xtline", "xtline_ex", "xtlogit",
+    "xtlogit_8", "xtlogit_d2", "xtlogit_fe_p", "xtlogit_pa_p",
+    "xtlogit_re_p", "xtmixed", "xtmixed_estat", "xtmixed_p",
+    "xtnb_fe", "xtnb_lf", "xtnbreg", "xtnbreg_pa_p",
+    "xtnbreg_refe_p", "xtpcse", "xtpcse_p", "xtpois", "xtpoisson",
+    "xtpoisson_d2", "xtpoisson_pa_p", "xtpoisson_refe_p", "xtpred",
+    "xtprobit", "xtprobit_8", "xtprobit_d2", "xtprobit_re_p",
+    "xtps_fe", "xtps_lf", "xtps_ren", "xtps_ren_8", "xtrar_p",
+    "xtrc", "xtrc_p", "xtrchh", "xtrefe_p", "xtreg", "xtreg_be",
+    "xtreg_fe", "xtreg_ml", "xtreg_pa_p", "xtreg_re",
+    "xtregar", "xtrere_p", "xtset", "xtsf_ll", "xtsf_llti",
+    "xtsum", "xttab", "xttest0", "xttobit", "xttobit_8",
+    "xttobit_p", "xttrans", "yx", "yxview__barlike_draw",
+    "yxview_area_draw", "yxview_bar_draw", "yxview_dot_draw",
+    "yxview_dropline_draw", "yxview_function_draw",
+    "yxview_iarrow_draw", "yxview_ilabels_draw",
+    "yxview_normal_draw", "yxview_pcarrow_draw",
+    "yxview_pcbarrow_draw", "yxview_pccapsym_draw",
+    "yxview_pcscatter_draw", "yxview_pcspike_draw",
+    "yxview_rarea_draw", "yxview_rbar_draw", "yxview_rbarm_draw",
+    "yxview_rcap_draw", "yxview_rcapsym_draw",
+    "yxview_rconnected_draw", "yxview_rline_draw",
+    "yxview_rscatter_draw", "yxview_rspike_draw",
+    "yxview_spike_draw", "yxview_sunflower_draw", "zap_s", "zinb",
+    "zinb_llf", "zinb_plf", "zip", "zip_llf", "zip_p", "zip_plf",
+    "zt_ct_5", "zt_hc_5", "zt_hcd_5", "zt_is_5", "zt_iss_5",
+    "zt_sho_5", "zt_smp_5", "ztbase_5", "ztcox_5", "ztdes_5",
+    "ztereg_5", "ztfill_5", "ztgen_5", "ztir_5", "ztjoin_5", "ztnb",
+    "ztnb_p", "ztp", "ztp_p", "zts_5", "ztset_5", "ztspli_5",
+    "ztsum_5", "zttoct_5", "ztvary_5", "ztweib_5"
+)
+
+builtins_functions = (
+    "Cdhms", "Chms", "Clock", "Cmdyhms", "Cofc", "Cofd", "F",
+    "Fden", "Ftail", "I", "J", "_caller", "abbrev", "abs", "acos",
+    "acosh", "asin", "asinh", "atan", "atan2", "atanh",
+    "autocode", "betaden", "binomial", "binomialp", "binomialtail",
+    "binormal", "bofd", "byteorder", "c", "ceil", "char",
+    "chi2", "chi2den", "chi2tail", "cholesky", "chop", "clip",
+    "clock", "cloglog", "cofC", "cofd", "colnumb", "colsof", "comb",
+    "cond", "corr", "cos", "cosh", "d", "daily", "date", "day",
+    "det", "dgammapda", "dgammapdada", "dgammapdadx", "dgammapdx",
+    "dgammapdxdx", "dhms", "diag", "diag0cnt", "digamma",
+    "dofC", "dofb", "dofc", "dofh", "dofm", "dofq", "dofw",
+    "dofy", "dow", "doy", "dunnettprob", "e", "el", "epsdouble",
+    "epsfloat", "exp", "fileexists", "fileread", "filereaderror",
+    "filewrite", "float", "floor", "fmtwidth", "gammaden",
+    "gammap", "gammaptail", "get", "group", "h", "hadamard",
+    "halfyear", "halfyearly", "has_eprop", "hh", "hhC", "hms",
+    "hofd", "hours", "hypergeometric", "hypergeometricp", "ibeta",
+    "ibetatail", "index", "indexnot", "inlist", "inrange", "int",
+    "inv", "invF", "invFtail", "invbinomial", "invbinomialtail",
+    "invchi2", "invchi2tail", "invcloglog", "invdunnettprob",
+    "invgammap", "invgammaptail", "invibeta", "invibetatail",
+    "invlogit", "invnFtail", "invnbinomial", "invnbinomialtail",
+    "invnchi2", "invnchi2tail", "invnibeta", "invnorm", "invnormal",
+    "invnttail", "invpoisson", "invpoissontail", "invsym", "invt",
+    "invttail", "invtukeyprob", "irecode", "issym", "issymmetric",
+    "itrim", "length", "ln", "lnfact", "lnfactorial", "lngamma",
+    "lnnormal", "lnnormalden", "log", "log10", "logit", "lower",
+    "ltrim", "m", "match", "matmissing", "matrix", "matuniform",
+    "max", "maxbyte", "maxdouble", "maxfloat", "maxint", "maxlong",
+    "mdy", "mdyhms", "mi", "min", "minbyte", "mindouble",
+    "minfloat", "minint", "minlong", "minutes", "missing", "mm",
+    "mmC", "mod", "mofd", "month", "monthly", "mreldif",
+    "msofhours", "msofminutes", "msofseconds", "nF", "nFden",
+    "nFtail", "nbetaden", "nbinomial", "nbinomialp", "nbinomialtail",
+    "nchi2", "nchi2den", "nchi2tail", "nibeta", "norm", "normal",
+    "normalden", "normd", "npnF", "npnchi2", "npnt", "nt", "ntden",
+    "nttail", "nullmat", "plural", "poisson", "poissonp",
+    "poissontail", "proper", "q", "qofd", "quarter", "quarterly",
+    "r", "rbeta", "rbinomial", "rchi2", "real", "recode", "regexm",
+    "regexr", "regexs", "reldif", "replay", "return", "reverse",
+    "rgamma", "rhypergeometric", "rnbinomial", "rnormal", "round",
+    "rownumb", "rowsof", "rpoisson", "rt", "rtrim", "runiform", "s",
+    "scalar", "seconds", "sign", "sin", "sinh", "smallestdouble",
+    "soundex", "soundex_nara", "sqrt", "ss", "ssC", "strcat",
+    "strdup", "string", "strlen", "strlower", "strltrim", "strmatch",
+    "strofreal", "strpos", "strproper", "strreverse", "strrtrim",
+    "strtoname", "strtrim", "strupper", "subinstr", "subinword",
+    "substr", "sum", "sweep", "syminv", "t", "tC", "tan", "tanh",
+    "tc", "td", "tden", "th", "tin", "tm", "tq", "trace",
+    "trigamma", "trim", "trunc", "ttail", "tukeyprob", "tw",
+    "twithin", "uniform", "upper", "vec", "vecdiag", "w", "week",
+    "weekly", "wofd", "word", "wordcount", "year", "yearly",
+    "yh", "ym", "yofd", "yq", "yw"
+)
+
+

packages/wakatime/packages/pygments/lexers/_vim_builtins.py

@@ -5,7 +5,7 @@
 
     This file is autogenerated by scripts/get_vimkw.py
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/actionscript.py

@@ -5,7 +5,7 @@
 
     Lexers for ActionScript and MXML.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/agile.py

@@ -5,7 +5,7 @@
 
     Just export lexer classes previously contained in this module.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/algebra.py

@@ -5,7 +5,7 @@
 
     Lexers for computer algebra systems.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -15,7 +15,7 @@ from pygments.lexer import RegexLexer, bygroups, words
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
     Number, Punctuation
 
-__all__ = ['GAPLexer', 'MathematicaLexer', 'MuPADLexer']
+__all__ = ['GAPLexer', 'MathematicaLexer', 'MuPADLexer', 'BCLexer']
 
 
 class GAPLexer(RegexLexer):
@@ -65,7 +65,7 @@ class GAPLexer(RegexLexer):
             (r'[0-9]+(?:\.[0-9]*)?(?:e[0-9]+)?', Number),
             (r'\.[0-9]+(?:e[0-9]+)?', Number),
             (r'.', Text)
-        ]
+        ],
     }
 
 
@@ -104,9 +104,9 @@ class MathematicaLexer(RegexLexer):
             (r'#\d*', Name.Variable),
             (r'([a-zA-Z]+[a-zA-Z0-9]*)', Name),
 
-            (r'-?[0-9]+\.[0-9]*', Number.Float),
-            (r'-?[0-9]*\.[0-9]+', Number.Float),
-            (r'-?[0-9]+', Number.Integer),
+            (r'-?\d+\.\d*', Number.Float),
+            (r'-?\d*\.\d+', Number.Float),
+            (r'-?\d+', Number.Integer),
 
             (words(operators), Operator),
             (words(punctuation), Punctuation),
@@ -183,5 +183,39 @@ class MuPADLexer(RegexLexer):
             (r'/\*', Comment.Multiline, '#push'),
             (r'\*/', Comment.Multiline, '#pop'),
             (r'[*/]', Comment.Multiline)
-        ]
+        ],
+    }
+
+
+class BCLexer(RegexLexer):
+    """
+    A `BC <https://www.gnu.org/software/bc/>`_ lexer.
+
+    .. versionadded:: 2.1
+    """
+    name = 'BC'
+    aliases = ['bc']
+    filenames = ['*.bc']
+
+    tokens = {
+        'root': [
+            (r'/\*', Comment.Multiline, 'comment'),
+            (r'"(?:[^"\\]|\\.)*"', String),
+            (r'[{}();,]', Punctuation),
+            (words(('if', 'else', 'while', 'for', 'break', 'continue',
+                    'halt', 'return', 'define', 'auto', 'print', 'read',
+                    'length', 'scale', 'sqrt', 'limits', 'quit',
+                    'warranty'), suffix=r'\b'), Keyword),
+            (r'\+\+|--|\|\||&&|'
+             r'([-<>+*%\^/!=])=?', Operator),
+            # bc doesn't support exponential
+            (r'[0-9]+(\.[0-9]*)?', Number),
+            (r'\.[0-9]+', Number),
+            (r'.', Text)
+        ],
+        'comment': [
+            (r'[^*/]+', Comment.Multiline),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'[*/]', Comment.Multiline)
+        ],
     }

packages/wakatime/packages/pygments/lexers/ambient.py

@@ -5,7 +5,7 @@
 
     Lexers for AmbientTalk language.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/ampl.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.ampl
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the ampl language. <http://ampl.com/>
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, using, this, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['AmplLexer']
+
+
+class AmplLexer(RegexLexer):
+    """
+    For AMPL source code.
+
+    .. versionadded:: 2.2
+    """
+    name = 'Ampl'
+    aliases = ['ampl']
+    filenames = ['*.run']
+
+    tokens = {
+        'root': [
+            (r'\n', Text),
+            (r'\s+', Text.Whitespace),
+            (r'#.*?\n', Comment.Single),
+            (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+            (words((
+                'call', 'cd', 'close', 'commands', 'data', 'delete', 'display',
+                'drop', 'end', 'environ', 'exit', 'expand', 'include', 'load',
+                'model', 'objective', 'option', 'problem', 'purge', 'quit',
+                'redeclare', 'reload', 'remove', 'reset', 'restore', 'shell',
+                'show', 'solexpand', 'solution', 'solve', 'update', 'unload',
+                'xref', 'coeff', 'coef', 'cover', 'obj', 'interval', 'default',
+                'from', 'to', 'to_come', 'net_in', 'net_out', 'dimen',
+                'dimension', 'check', 'complements', 'write', 'function',
+                'pipe', 'format', 'if', 'then', 'else', 'in', 'while', 'repeat',
+                'for'), suffix=r'\b'), Keyword.Reserved),
+            (r'(integer|binary|symbolic|ordered|circular|reversed|INOUT|IN|OUT|LOCAL)',
+             Keyword.Type),
+            (r'\".*?\"', String.Double),
+            (r'\'.*?\'', String.Single),
+            (r'[()\[\]{},;:]+', Punctuation),
+            (r'\b(\w+)(\.)(astatus|init0|init|lb0|lb1|lb2|lb|lrc|'
+             r'lslack|rc|relax|slack|sstatus|status|ub0|ub1|ub2|'
+             r'ub|urc|uslack|val)',
+             bygroups(Name.Variable, Punctuation, Keyword.Reserved)),
+            (r'(set|param|var|arc|minimize|maximize|subject to|s\.t\.|subj to|'
+             r'node|table|suffix|read table|write table)(\s+)(\w+)',
+             bygroups(Keyword.Declaration, Text, Name.Variable)),
+            (r'(param)(\s*)(:)(\s*)(\w+)(\s*)(:)(\s*)((\w|\s)+)',
+             bygroups(Keyword.Declaration, Text, Punctuation, Text,
+                      Name.Variable, Text, Punctuation, Text, Name.Variable)),
+            (r'(let|fix|unfix)(\s*)((?:\{.*\})?)(\s*)(\w+)',
+             bygroups(Keyword.Declaration, Text, using(this), Text, Name.Variable)),
+            (words((
+                'abs', 'acos', 'acosh', 'alias', 'asin', 'asinh', 'atan', 'atan2',
+                'atanh', 'ceil', 'ctime', 'cos', 'exp', 'floor', 'log', 'log10',
+                'max', 'min', 'precision', 'round', 'sin', 'sinh', 'sqrt', 'tan',
+                'tanh', 'time', 'trunc', 'Beta', 'Cauchy', 'Exponential', 'Gamma',
+                'Irand224', 'Normal', 'Normal01', 'Poisson', 'Uniform', 'Uniform01',
+                'num', 'num0', 'ichar', 'char', 'length', 'substr', 'sprintf',
+                'match', 'sub', 'gsub', 'print', 'printf', 'next', 'nextw', 'prev',
+                'prevw', 'first', 'last', 'ord', 'ord0', 'card', 'arity',
+                'indexarity'), prefix=r'\b', suffix=r'\b'), Name.Builtin),
+            (r'(\+|\-|\*|/|\*\*|=|<=|>=|==|\||\^|<|>|\!|\.\.|:=|\&|\!=|<<|>>)',
+             Operator),
+            (words((
+                'or', 'exists', 'forall', 'and', 'in', 'not', 'within', 'union',
+                'diff', 'difference', 'symdiff', 'inter', 'intersect',
+                'intersection', 'cross', 'setof', 'by', 'less', 'sum', 'prod',
+                'product', 'div', 'mod'), suffix=r'\b'),
+             Keyword.Reserved),  # Operator.Name but not enough emphasized with that
+            (r'(\d+\.(?!\.)\d*|\.(?!.)\d+)([eE][+-]?\d+)?', Number.Float),
+            (r'\d+([eE][+-]?\d+)?', Number.Integer),
+            (r'[+-]?Infinity', Number.Integer),
+            (r'(\w+|(\.(?!\.)))', Text)
+        ]
+
+    }

packages/wakatime/packages/pygments/lexers/apl.py

@@ -5,7 +5,7 @@
 
     Lexers for APL.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/archetype.py

@@ -0,0 +1,318 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.archetype
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Archetype-related syntaxes, including:
+
+    - ODIN syntax <https://github.com/openEHR/odin>
+    - ADL syntax <http://www.openehr.org/releases/trunk/architecture/am/adl2.pdf>
+    - cADL sub-syntax of ADL
+
+    For uses of this syntax, see the openEHR archetypes <http://www.openEHR.org/ckm>
+
+    Contributed by Thomas Beale <https://github.com/wolandscat>,
+    <https://bitbucket.org/thomas_beale>.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, bygroups, using, default
+from pygments.token import Text, Comment, Name, Literal, Number, String, \
+    Punctuation, Keyword, Operator, Generic
+
+__all__ = ['OdinLexer', 'CadlLexer', 'AdlLexer']
+
+
+class AtomsLexer(RegexLexer):
+    """
+    Lexer for Values used in ADL and ODIN.
+
+    .. versionadded:: 2.1
+    """
+
+    tokens = {
+        # ----- pseudo-states for inclusion -----
+        'whitespace': [
+            (r'\n', Text),
+            (r'\s+', Text),
+            (r'[ \t]*--.*$', Comment),
+        ],
+        'archetype_id': [
+            (r'[ \t]*([a-zA-Z]\w+(\.[a-zA-Z]\w+)*::)?[a-zA-Z]\w+(-[a-zA-Z]\w+){2}'
+             r'\.\w+[\w-]*\.v\d+(\.\d+){,2}((-[a-z]+)(\.\d+)?)?', Name.Decorator),
+        ],
+        'date_constraints': [
+            # ISO 8601-based date/time constraints
+            (r'[Xx?YyMmDdHhSs\d]{2,4}([:-][Xx?YyMmDdHhSs\d]{2}){2}', Literal.Date),
+            # ISO 8601-based duration constraints + optional trailing slash
+            (r'(P[YyMmWwDd]+(T[HhMmSs]+)?|PT[HhMmSs]+)/?', Literal.Date),
+        ],
+        'ordered_values': [
+            # ISO 8601 date with optional 'T' ligature
+            (r'\d{4}-\d{2}-\d{2}T?', Literal.Date),
+            # ISO 8601 time
+            (r'\d{2}:\d{2}:\d{2}(\.\d+)?([+-]\d{4}|Z)?', Literal.Date),
+            # ISO 8601 duration
+            (r'P((\d*(\.\d+)?[YyMmWwDd]){1,3}(T(\d*(\.\d+)?[HhMmSs]){,3})?|'
+             r'T(\d*(\.\d+)?[HhMmSs]){,3})', Literal.Date),
+            (r'[+-]?(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+', Number.Float),
+            (r'[+-]?(\d+)*\.\d+%?', Number.Float),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
+            (r'[+-]?\d+%?', Number.Integer),
+        ],
+        'values': [
+            include('ordered_values'),
+            (r'([Tt]rue|[Ff]alse)', Literal),
+            (r'"', String, 'string'),
+            (r"'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char),
+            (r'[a-z][a-z0-9+.-]*:', Literal, 'uri'),
+            # term code
+            (r'(\[)(\w[\w-]*(?:\([^)\n]+\))?)(::)(\w[\w-]*)(\])',
+             bygroups(Punctuation, Name.Decorator, Punctuation, Name.Decorator,
+                      Punctuation)),
+            (r'\|', Punctuation, 'interval'),
+            # list continuation
+            (r'\.\.\.', Punctuation),
+        ],
+        'constraint_values': [
+            (r'(\[)(\w[\w-]*(?:\([^)\n]+\))?)(::)',
+             bygroups(Punctuation, Name.Decorator, Punctuation), 'adl14_code_constraint'),
+            # ADL 1.4 ordinal constraint
+            (r'(\d*)(\|)(\[\w[\w-]*::\w[\w-]*\])((?:[,;])?)',
+             bygroups(Number, Punctuation, Name.Decorator, Punctuation)),
+            include('date_constraints'),
+            include('values'),
+        ],
+
+        # ----- real states -----
+        'string': [
+            ('"', String, '#pop'),
+            (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|'
+             r'u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|[0-7]{1,3})', String.Escape),
+            # all other characters
+            (r'[^\\"]+', String),
+            # stray backslash
+            (r'\\', String),
+        ],
+        'uri': [
+            # effective URI terminators
+            (r'[,>\s]', Punctuation, '#pop'),
+            (r'[^>\s,]+', Literal),
+        ],
+        'interval': [
+            (r'\|', Punctuation, '#pop'),
+            include('ordered_values'),
+            (r'\.\.', Punctuation),
+            (r'[<>=] *', Punctuation),
+            # handle +/-
+            (r'\+/-', Punctuation),
+            (r'\s+', Text),
+        ],
+        'any_code': [
+            include('archetype_id'),
+            # if it is a code
+            (r'[a-z_]\w*[0-9.]+(@[^\]]+)?', Name.Decorator),
+            # if it is tuple with attribute names
+            (r'[a-z_]\w*', Name.Class),
+            # if it is an integer, i.e. Xpath child index
+            (r'[0-9]+', Text),
+            (r'\|', Punctuation, 'code_rubric'),
+            (r'\]', Punctuation, '#pop'),
+            # handle use_archetype statement
+            (r'\s*,\s*', Punctuation),
+        ],
+        'code_rubric': [
+            (r'\|', Punctuation, '#pop'),
+            (r'[^|]+', String),
+        ],
+        'adl14_code_constraint': [
+            (r'\]', Punctuation, '#pop'),
+            (r'\|', Punctuation, 'code_rubric'),
+            (r'(\w[\w-]*)([;,]?)', bygroups(Name.Decorator, Punctuation)),
+            include('whitespace'),
+        ],
+    }
+
+
+class OdinLexer(AtomsLexer):
+    """
+    Lexer for ODIN syntax.
+
+    .. versionadded:: 2.1
+    """
+    name = 'ODIN'
+    aliases = ['odin']
+    filenames = ['*.odin']
+    mimetypes = ['text/odin']
+
+    tokens = {
+        'path': [
+            (r'>', Punctuation, '#pop'),
+            # attribute name
+            (r'[a-z_]\w*', Name.Class),
+            (r'/', Punctuation),
+            (r'\[', Punctuation, 'key'),
+            (r'\s*,\s*', Punctuation, '#pop'),
+            (r'\s+', Text, '#pop'),
+        ],
+        'key': [
+            include('values'),
+            (r'\]', Punctuation, '#pop'),
+        ],
+        'type_cast': [
+            (r'\)', Punctuation, '#pop'),
+            (r'[^)]+',  Name.Class),
+        ],
+        'root': [
+            include('whitespace'),
+            (r'([Tt]rue|[Ff]alse)', Literal),
+            include('values'),
+            # x-ref path
+            (r'/', Punctuation, 'path'),
+            # x-ref path starting with key
+            (r'\[', Punctuation, 'key'),
+            # attribute name
+            (r'[a-z_]\w*', Name.Class),
+            (r'=', Operator),
+            (r'\(', Punctuation, 'type_cast'),
+            (r',', Punctuation),
+            (r'<', Punctuation),
+            (r'>', Punctuation),
+            (r';', Punctuation),
+        ],
+    }
+
+
+class CadlLexer(AtomsLexer):
+    """
+    Lexer for cADL syntax.
+
+    .. versionadded:: 2.1
+    """
+    name = 'cADL'
+    aliases = ['cadl']
+    filenames = ['*.cadl']
+
+    tokens = {
+        'path': [
+            # attribute name
+            (r'[a-z_]\w*', Name.Class),
+            (r'/', Punctuation),
+            (r'\[', Punctuation, 'any_code'),
+            (r'\s+', Punctuation, '#pop'),
+        ],
+        'root': [
+            include('whitespace'),
+            (r'(cardinality|existence|occurrences|group|include|exclude|'
+             r'allow_archetype|use_archetype|use_node)\W', Keyword.Type),
+            (r'(and|or|not|there_exists|xor|implies|for_all)\W', Keyword.Type),
+            (r'(after|before|closed)\W', Keyword.Type),
+            (r'(not)\W', Operator),
+            (r'(matches|is_in)\W', Operator),
+            # is_in / not is_in char
+            (u'(\u2208|\u2209)', Operator),
+            # there_exists / not there_exists / for_all / and / or
+            (u'(\u2203|\u2204|\u2200|\u2227|\u2228|\u22BB|\223C)',
+             Operator),
+            # regex in slot or as string constraint
+            (r'(\{)(\s*/[^}]+/\s*)(\})',
+             bygroups(Punctuation, String.Regex, Punctuation)),
+            # regex in slot or as string constraint
+            (r'(\{)(\s*\^[^}]+\^\s*)(\})',
+             bygroups(Punctuation, String.Regex, Punctuation)),
+            (r'/', Punctuation, 'path'),
+            # for cardinality etc
+            (r'(\{)((?:\d+\.\.)?(?:\d+|\*))'
+             r'((?:\s*;\s*(?:ordered|unordered|unique)){,2})(\})',
+             bygroups(Punctuation, Number, Number, Punctuation)),
+            # [{ is start of a tuple value
+            (r'\[\{', Punctuation),
+            (r'\}\]', Punctuation),
+            (r'\{', Punctuation),
+            (r'\}', Punctuation),
+            include('constraint_values'),
+            # type name
+            (r'[A-Z]\w+(<[A-Z]\w+([A-Za-z_<>]*)>)?',  Name.Class),
+            # attribute name
+            (r'[a-z_]\w*', Name.Class),
+            (r'\[', Punctuation, 'any_code'),
+            (r'(~|//|\\\\|\+|-|/|\*|\^|!=|=|<=|>=|<|>]?)', Operator),
+            (r'\(', Punctuation),
+            (r'\)', Punctuation),
+            # for lists of values
+            (r',', Punctuation),
+            (r'"', String, 'string'),
+            # for assumed value
+            (r';', Punctuation),
+        ],
+    }
+
+
+class AdlLexer(AtomsLexer):
+    """
+    Lexer for ADL syntax.
+
+    .. versionadded:: 2.1
+    """
+
+    name = 'ADL'
+    aliases = ['adl']
+    filenames = ['*.adl', '*.adls', '*.adlf', '*.adlx']
+
+    tokens = {
+        'whitespace': [
+            # blank line ends
+            (r'\s*\n', Text),
+            # comment-only line
+            (r'^[ \t]*--.*$', Comment),
+        ],
+        'odin_section': [
+            # repeating the following two rules from the root state enable multi-line
+            # strings that start in the first column to be dealt with
+            (r'^(language|description|ontology|terminology|annotations|'
+             r'component_terminologies|revision_history)[ \t]*\n', Generic.Heading),
+            (r'^(definition)[ \t]*\n', Generic.Heading, 'cadl_section'),
+            (r'^([ \t]*|[ \t]+.*)\n', using(OdinLexer)),
+            (r'^([^"]*")(>[ \t]*\n)', bygroups(String, Punctuation)),
+            # template overlay delimiter
+            (r'^----------*\n', Text, '#pop'),
+            (r'^.*\n', String),
+            default('#pop'),
+        ],
+        'cadl_section': [
+            (r'^([ \t]*|[ \t]+.*)\n', using(CadlLexer)),
+            default('#pop'),
+        ],
+        'rules_section': [
+            (r'^[ \t]+.*\n', using(CadlLexer)),
+            default('#pop'),
+        ],
+        'metadata': [
+            (r'\)', Punctuation, '#pop'),
+            (r';', Punctuation),
+            (r'([Tt]rue|[Ff]alse)', Literal),
+            # numbers and version ids
+            (r'\d+(\.\d+)*', Literal),
+            # Guids
+            (r'(\d|[a-fA-F])+(-(\d|[a-fA-F])+){3,}', Literal),
+            (r'\w+', Name.Class),
+            (r'"', String, 'string'),
+            (r'=', Operator),
+            (r'[ \t]+', Text),
+            default('#pop'),
+        ],
+        'root': [
+            (r'^(archetype|template_overlay|operational_template|template|'
+             r'speciali[sz]e)', Generic.Heading),
+            (r'^(language|description|ontology|terminology|annotations|'
+             r'component_terminologies|revision_history)[ \t]*\n',
+             Generic.Heading, 'odin_section'),
+            (r'^(definition)[ \t]*\n', Generic.Heading, 'cadl_section'),
+            (r'^(rules)[ \t]*\n', Generic.Heading, 'rules_section'),
+            include('archetype_id'),
+            (r'[ \t]*\(', Punctuation, 'metadata'),
+            include('whitespace'),
+        ],
+    }

packages/wakatime/packages/pygments/lexers/asm.py

@@ -0,0 +1,641 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.asm
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for assembly languages.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using, words, \
+    DelegatingLexer
+from pygments.lexers.c_cpp import CppLexer, CLexer
+from pygments.lexers.d import DLexer
+from pygments.token import Text, Name, Number, String, Comment, Punctuation, \
+    Other, Keyword, Operator
+
+__all__ = ['GasLexer', 'ObjdumpLexer', 'DObjdumpLexer', 'CppObjdumpLexer',
+           'CObjdumpLexer', 'HsailLexer', 'LlvmLexer', 'NasmLexer',
+           'NasmObjdumpLexer', 'TasmLexer', 'Ca65Lexer']
+
+
+class GasLexer(RegexLexer):
+    """
+    For Gas (AT&T) assembly code.
+    """
+    name = 'GAS'
+    aliases = ['gas', 'asm']
+    filenames = ['*.s', '*.S']
+    mimetypes = ['text/x-gas']
+
+    #: optional Comment or Whitespace
+    string = r'"(\\"|[^"])*"'
+    char = r'[\w$.@-]'
+    identifier = r'(?:[a-zA-Z$_]' + char + '*|\.' + char + '+)'
+    number = r'(?:0[xX][a-zA-Z0-9]+|\d+)'
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            (identifier + ':', Name.Label),
+            (r'\.' + identifier, Name.Attribute, 'directive-args'),
+            (r'lock|rep(n?z)?|data\d+', Name.Attribute),
+            (identifier, Name.Function, 'instruction-args'),
+            (r'[\r\n]+', Text)
+        ],
+        'directive-args': [
+            (identifier, Name.Constant),
+            (string, String),
+            ('@' + identifier, Name.Attribute),
+            (number, Number.Integer),
+            (r'[\r\n]+', Text, '#pop'),
+
+            include('punctuation'),
+            include('whitespace')
+        ],
+        'instruction-args': [
+            # For objdump-disassembled code, shouldn't occur in
+            # actual assembler input
+            ('([a-z0-9]+)( )(<)('+identifier+')(>)',
+                bygroups(Number.Hex, Text, Punctuation, Name.Constant,
+                         Punctuation)),
+            ('([a-z0-9]+)( )(<)('+identifier+')([-+])('+number+')(>)',
+                bygroups(Number.Hex, Text, Punctuation, Name.Constant,
+                         Punctuation, Number.Integer, Punctuation)),
+
+            # Address constants
+            (identifier, Name.Constant),
+            (number, Number.Integer),
+            # Registers
+            ('%' + identifier, Name.Variable),
+            # Numeric constants
+            ('$'+number, Number.Integer),
+            (r"$'(.|\\')'", String.Char),
+            (r'[\r\n]+', Text, '#pop'),
+
+            include('punctuation'),
+            include('whitespace')
+        ],
+        'whitespace': [
+            (r'\n', Text),
+            (r'\s+', Text),
+            (r'[;#].*?\n', Comment)
+        ],
+        'punctuation': [
+            (r'[-*,.()\[\]!:]+', Punctuation)
+        ]
+    }
+
+    def analyse_text(text):
+        if re.match(r'^\.(text|data|section)', text, re.M):
+            return True
+        elif re.match(r'^\.\w+', text, re.M):
+            return 0.1
+
+
+def _objdump_lexer_tokens(asm_lexer):
+    """
+    Common objdump lexer tokens to wrap an ASM lexer.
+    """
+    hex_re = r'[0-9A-Za-z]'
+    return {
+        'root': [
+            # File name & format:
+            ('(.*?)(:)( +file format )(.*?)$',
+                bygroups(Name.Label, Punctuation, Text, String)),
+            # Section header
+            ('(Disassembly of section )(.*?)(:)$',
+                bygroups(Text, Name.Label, Punctuation)),
+            # Function labels
+            # (With offset)
+            ('('+hex_re+'+)( )(<)(.*?)([-+])(0[xX][A-Za-z0-9]+)(>:)$',
+                bygroups(Number.Hex, Text, Punctuation, Name.Function,
+                         Punctuation, Number.Hex, Punctuation)),
+            # (Without offset)
+            ('('+hex_re+'+)( )(<)(.*?)(>:)$',
+                bygroups(Number.Hex, Text, Punctuation, Name.Function,
+                         Punctuation)),
+            # Code line with disassembled instructions
+            ('( *)('+hex_re+r'+:)(\t)((?:'+hex_re+hex_re+' )+)( *\t)([a-zA-Z].*?)$',
+                bygroups(Text, Name.Label, Text, Number.Hex, Text,
+                         using(asm_lexer))),
+            # Code line with ascii
+            ('( *)('+hex_re+r'+:)(\t)((?:'+hex_re+hex_re+' )+)( *)(.*?)$',
+                bygroups(Text, Name.Label, Text, Number.Hex, Text, String)),
+            # Continued code line, only raw opcodes without disassembled
+            # instruction
+            ('( *)('+hex_re+r'+:)(\t)((?:'+hex_re+hex_re+' )+)$',
+                bygroups(Text, Name.Label, Text, Number.Hex)),
+            # Skipped a few bytes
+            (r'\t\.\.\.$', Text),
+            # Relocation line
+            # (With offset)
+            (r'(\t\t\t)('+hex_re+r'+:)( )([^\t]+)(\t)(.*?)([-+])(0x'+hex_re+'+)$',
+                bygroups(Text, Name.Label, Text, Name.Property, Text,
+                         Name.Constant, Punctuation, Number.Hex)),
+            # (Without offset)
+            (r'(\t\t\t)('+hex_re+r'+:)( )([^\t]+)(\t)(.*?)$',
+                bygroups(Text, Name.Label, Text, Name.Property, Text,
+                         Name.Constant)),
+            (r'[^\n]+\n', Other)
+        ]
+    }
+
+
+class ObjdumpLexer(RegexLexer):
+    """
+    For the output of 'objdump -dr'
+    """
+    name = 'objdump'
+    aliases = ['objdump']
+    filenames = ['*.objdump']
+    mimetypes = ['text/x-objdump']
+
+    tokens = _objdump_lexer_tokens(GasLexer)
+
+
+class DObjdumpLexer(DelegatingLexer):
+    """
+    For the output of 'objdump -Sr on compiled D files'
+    """
+    name = 'd-objdump'
+    aliases = ['d-objdump']
+    filenames = ['*.d-objdump']
+    mimetypes = ['text/x-d-objdump']
+
+    def __init__(self, **options):
+        super(DObjdumpLexer, self).__init__(DLexer, ObjdumpLexer, **options)
+
+
+class CppObjdumpLexer(DelegatingLexer):
+    """
+    For the output of 'objdump -Sr on compiled C++ files'
+    """
+    name = 'cpp-objdump'
+    aliases = ['cpp-objdump', 'c++-objdumb', 'cxx-objdump']
+    filenames = ['*.cpp-objdump', '*.c++-objdump', '*.cxx-objdump']
+    mimetypes = ['text/x-cpp-objdump']
+
+    def __init__(self, **options):
+        super(CppObjdumpLexer, self).__init__(CppLexer, ObjdumpLexer, **options)
+
+
+class CObjdumpLexer(DelegatingLexer):
+    """
+    For the output of 'objdump -Sr on compiled C files'
+    """
+    name = 'c-objdump'
+    aliases = ['c-objdump']
+    filenames = ['*.c-objdump']
+    mimetypes = ['text/x-c-objdump']
+
+    def __init__(self, **options):
+        super(CObjdumpLexer, self).__init__(CLexer, ObjdumpLexer, **options)
+
+
+class HsailLexer(RegexLexer):
+    """
+    For HSAIL assembly code.
+
+    .. versionadded:: 2.2
+    """
+    name = 'HSAIL'
+    aliases = ['hsail', 'hsa']
+    filenames = ['*.hsail']
+    mimetypes = ['text/x-hsail']
+
+    string = r'"[^"]*?"'
+    identifier = r'[a-zA-Z_][\w.]*'
+    # Registers
+    register_number = r'[0-9]+'
+    register = r'(\$(c|s|d|q)' + register_number + ')'
+    # Qualifiers
+    alignQual = r'(align\(\d+\))'
+    widthQual = r'(width\((\d+|all)\))'
+    allocQual = r'(alloc\(agent\))'
+    # Instruction Modifiers
+    roundingMod = (r'((_ftz)?(_up|_down|_zero|_near))')
+    datatypeMod = (r'_('
+                   # packedTypes
+                   r'u8x4|s8x4|u16x2|s16x2|u8x8|s8x8|u16x4|s16x4|u32x2|s32x2|'
+                   r'u8x16|s8x16|u16x8|s16x8|u32x4|s32x4|u64x2|s64x2|'
+                   r'f16x2|f16x4|f16x8|f32x2|f32x4|f64x2|'
+                   # baseTypes
+                   r'u8|s8|u16|s16|u32|s32|u64|s64|'
+                   r'b128|b8|b16|b32|b64|b1|'
+                   r'f16|f32|f64|'
+                   # opaqueType
+                   r'roimg|woimg|rwimg|samp|sig32|sig64)')
+
+    # Numeric Constant
+    float = r'((\d+\.)|(\d*\.\d+))[eE][+-]?\d+'
+    hexfloat = r'0[xX](([0-9a-fA-F]+\.[0-9a-fA-F]*)|([0-9a-fA-F]*\.[0-9a-fA-F]+))[pP][+-]?\d+'
+    ieeefloat = r'0((h|H)[0-9a-fA-F]{4}|(f|F)[0-9a-fA-F]{8}|(d|D)[0-9a-fA-F]{16})'
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            include('comments'),
+
+            (string, String),
+
+            (r'@' + identifier + ':?', Name.Label),
+
+            (register, Name.Variable.Anonymous),
+
+            include('keyword'),
+
+            (r'&' + identifier, Name.Variable.Global),
+            (r'%' + identifier, Name.Variable),
+
+            (hexfloat, Number.Hex),
+            (r'0[xX][a-fA-F0-9]+', Number.Hex),
+            (ieeefloat, Number.Float),
+            (float, Number.Float),
+            ('\d+', Number.Integer),
+
+            (r'[=<>{}\[\]()*.,:;!]|x\b', Punctuation)
+        ],
+        'whitespace': [
+            (r'(\n|\s)+', Text),
+        ],
+        'comments': [
+            (r'/\*.*?\*/', Comment.Multiline),
+            (r'//.*?\n', Comment.Single),
+        ],
+        'keyword': [
+            # Types
+            (r'kernarg' + datatypeMod, Keyword.Type),
+
+            # Regular keywords
+            (r'\$(full|base|small|large|default|zero|near)', Keyword),
+            (words((
+                'module', 'extension', 'pragma', 'prog', 'indirect', 'signature',
+                'decl', 'kernel', 'function', 'enablebreakexceptions',
+                'enabledetectexceptions', 'maxdynamicgroupsize', 'maxflatgridsize',
+                'maxflatworkgroupsize', 'requireddim', 'requiredgridsize',
+                'requiredworkgroupsize', 'requirenopartialworkgroups'),
+                suffix=r'\b'), Keyword),
+
+            # instructions
+            (roundingMod, Keyword),
+            (datatypeMod, Keyword),
+            (r'_(' + alignQual + '|' + widthQual + ')', Keyword),
+            (r'_kernarg', Keyword),
+            (r'(nop|imagefence)\b', Keyword),
+            (words((
+                'cleardetectexcept', 'clock', 'cuid', 'debugtrap', 'dim',
+                'getdetectexcept', 'groupbaseptr', 'kernargbaseptr', 'laneid',
+                'maxcuid', 'maxwaveid', 'packetid', 'setdetectexcept', 'waveid',
+                'workitemflatabsid', 'workitemflatid', 'nullptr', 'abs', 'bitrev',
+                'currentworkgroupsize', 'currentworkitemflatid', 'fract', 'ncos',
+                'neg', 'nexp2', 'nlog2', 'nrcp', 'nrsqrt', 'nsin', 'nsqrt',
+                'gridgroups', 'gridsize', 'not', 'sqrt', 'workgroupid',
+                'workgroupsize', 'workitemabsid', 'workitemid', 'ceil', 'floor',
+                'rint', 'trunc', 'add', 'bitmask', 'borrow', 'carry', 'copysign',
+                'div', 'rem', 'sub', 'shl', 'shr', 'and', 'or', 'xor', 'unpackhi',
+                'unpacklo', 'max', 'min', 'fma', 'mad', 'bitextract', 'bitselect',
+                'shuffle', 'cmov', 'bitalign', 'bytealign', 'lerp', 'nfma', 'mul',
+                'mulhi', 'mul24hi', 'mul24', 'mad24', 'mad24hi', 'bitinsert',
+                'combine', 'expand', 'lda', 'mov', 'pack', 'unpack', 'packcvt',
+                'unpackcvt', 'sad', 'sementp', 'ftos', 'stof', 'cmp', 'ld', 'st',
+                '_eq', '_ne', '_lt', '_le', '_gt', '_ge', '_equ', '_neu', '_ltu',
+                '_leu', '_gtu', '_geu', '_num', '_nan', '_seq', '_sne', '_slt',
+                '_sle', '_sgt', '_sge', '_snum', '_snan', '_sequ', '_sneu', '_sltu',
+                '_sleu', '_sgtu', '_sgeu', 'atomic', '_ld', '_st', '_cas', '_add',
+                '_and', '_exch', '_max', '_min', '_or', '_sub', '_wrapdec',
+                '_wrapinc', '_xor', 'ret', 'cvt', '_readonly', '_kernarg', '_global',
+                'br', 'cbr', 'sbr', '_scacq', '_screl', '_scar', '_rlx', '_wave',
+                '_wg', '_agent', '_system', 'ldimage', 'stimage', '_v2', '_v3', '_v4',
+                '_1d', '_2d', '_3d', '_1da', '_2da', '_1db', '_2ddepth', '_2dadepth',
+                '_width', '_height', '_depth', '_array', '_channelorder',
+                '_channeltype', 'querysampler', '_coord', '_filter', '_addressing',
+                'barrier', 'wavebarrier', 'initfbar', 'joinfbar', 'waitfbar',
+                'arrivefbar', 'leavefbar', 'releasefbar', 'ldf', 'activelaneid',
+                'activelanecount', 'activelanemask', 'activelanepermute', 'call',
+                'scall', 'icall', 'alloca', 'packetcompletionsig',
+                'addqueuewriteindex', 'casqueuewriteindex', 'ldqueuereadindex',
+                'stqueuereadindex', 'readonly', 'global', 'private', 'group',
+                'spill', 'arg', '_upi', '_downi', '_zeroi', '_neari', '_upi_sat',
+                '_downi_sat', '_zeroi_sat', '_neari_sat', '_supi', '_sdowni',
+                '_szeroi', '_sneari', '_supi_sat', '_sdowni_sat', '_szeroi_sat',
+                '_sneari_sat', '_pp', '_ps', '_sp', '_ss', '_s', '_p', '_pp_sat',
+                '_ps_sat', '_sp_sat', '_ss_sat', '_s_sat', '_p_sat')), Keyword),
+
+            # Integer types
+            (r'i[1-9]\d*', Keyword)
+        ]
+    }
+
+
+class LlvmLexer(RegexLexer):
+    """
+    For LLVM assembly code.
+    """
+    name = 'LLVM'
+    aliases = ['llvm']
+    filenames = ['*.ll']
+    mimetypes = ['text/x-llvm']
+
+    #: optional Comment or Whitespace
+    string = r'"[^"]*?"'
+    identifier = r'([-a-zA-Z$._][\w\-$.]*|' + string + ')'
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+
+            # Before keywords, because keywords are valid label names :(...
+            (identifier + '\s*:', Name.Label),
+
+            include('keyword'),
+
+            (r'%' + identifier, Name.Variable),
+            (r'@' + identifier, Name.Variable.Global),
+            (r'%\d+', Name.Variable.Anonymous),
+            (r'@\d+', Name.Variable.Global),
+            (r'#\d+', Name.Variable.Global),
+            (r'!' + identifier, Name.Variable),
+            (r'!\d+', Name.Variable.Anonymous),
+            (r'c?' + string, String),
+
+            (r'0[xX][a-fA-F0-9]+', Number),
+            (r'-?\d+(?:[.]\d+)?(?:[eE][-+]?\d+(?:[.]\d+)?)?', Number),
+
+            (r'[=<>{}\[\]()*.,!]|x\b', Punctuation)
+        ],
+        'whitespace': [
+            (r'(\n|\s)+', Text),
+            (r';.*?\n', Comment)
+        ],
+        'keyword': [
+            # Regular keywords
+            (words((
+                'begin', 'end', 'true', 'false', 'declare', 'define', 'global',
+                'constant', 'private', 'linker_private', 'internal',
+                'available_externally', 'linkonce', 'linkonce_odr', 'weak',
+                'weak_odr', 'appending', 'dllimport', 'dllexport', 'common',
+                'default', 'hidden', 'protected', 'extern_weak', 'external',
+                'thread_local', 'zeroinitializer', 'undef', 'null', 'to', 'tail',
+                'target', 'triple', 'datalayout', 'volatile', 'nuw', 'nsw', 'nnan',
+                'ninf', 'nsz', 'arcp', 'fast', 'exact', 'inbounds', 'align',
+                'addrspace', 'section', 'alias', 'module', 'asm', 'sideeffect',
+                'gc', 'dbg', 'linker_private_weak', 'attributes', 'blockaddress',
+                'initialexec', 'localdynamic', 'localexec', 'prefix', 'unnamed_addr',
+                'ccc', 'fastcc', 'coldcc', 'x86_stdcallcc', 'x86_fastcallcc',
+                'arm_apcscc', 'arm_aapcscc', 'arm_aapcs_vfpcc', 'ptx_device',
+                'ptx_kernel', 'intel_ocl_bicc', 'msp430_intrcc', 'spir_func',
+                'spir_kernel', 'x86_64_sysvcc', 'x86_64_win64cc', 'x86_thiscallcc',
+                'cc', 'c', 'signext', 'zeroext', 'inreg', 'sret', 'nounwind',
+                'noreturn', 'noalias', 'nocapture', 'byval', 'nest', 'readnone',
+                'readonly', 'inlinehint', 'noinline', 'alwaysinline', 'optsize', 'ssp',
+                'sspreq', 'noredzone', 'noimplicitfloat', 'naked', 'builtin', 'cold',
+                'nobuiltin', 'noduplicate', 'nonlazybind', 'optnone', 'returns_twice',
+                'sanitize_address', 'sanitize_memory', 'sanitize_thread', 'sspstrong',
+                'uwtable', 'returned', 'type', 'opaque', 'eq', 'ne', 'slt', 'sgt',
+                'sle', 'sge', 'ult', 'ugt', 'ule', 'uge', 'oeq', 'one', 'olt', 'ogt',
+                'ole', 'oge', 'ord', 'uno', 'ueq', 'une', 'x', 'acq_rel', 'acquire',
+                'alignstack', 'atomic', 'catch', 'cleanup', 'filter', 'inteldialect',
+                'max', 'min', 'monotonic', 'nand', 'personality', 'release', 'seq_cst',
+                'singlethread', 'umax', 'umin', 'unordered', 'xchg', 'add', 'fadd',
+                'sub', 'fsub', 'mul', 'fmul', 'udiv', 'sdiv', 'fdiv', 'urem', 'srem',
+                'frem', 'shl', 'lshr', 'ashr', 'and', 'or', 'xor', 'icmp', 'fcmp',
+                'phi', 'call', 'trunc', 'zext', 'sext', 'fptrunc', 'fpext', 'uitofp',
+                'sitofp', 'fptoui', 'fptosi', 'inttoptr', 'ptrtoint', 'bitcast',
+                'addrspacecast', 'select', 'va_arg', 'ret', 'br', 'switch', 'invoke',
+                'unwind', 'unreachable', 'indirectbr', 'landingpad', 'resume',
+                'malloc', 'alloca', 'free', 'load', 'store', 'getelementptr',
+                'extractelement', 'insertelement', 'shufflevector', 'getresult',
+                'extractvalue', 'insertvalue', 'atomicrmw', 'cmpxchg', 'fence',
+                'allocsize', 'amdgpu_cs', 'amdgpu_gs', 'amdgpu_kernel', 'amdgpu_ps',
+                'amdgpu_vs', 'any', 'anyregcc', 'argmemonly', 'avr_intrcc',
+                'avr_signalcc', 'caller', 'catchpad', 'catchret', 'catchswitch',
+                'cleanuppad', 'cleanupret', 'comdat', 'convergent', 'cxx_fast_tlscc',
+                'deplibs', 'dereferenceable', 'dereferenceable_or_null', 'distinct',
+                'exactmatch', 'externally_initialized', 'from', 'ghccc', 'hhvm_ccc',
+                'hhvmcc', 'ifunc', 'inaccessiblemem_or_argmemonly', 'inaccessiblememonly',
+                'inalloca', 'jumptable', 'largest', 'local_unnamed_addr', 'minsize',
+                'musttail', 'noduplicates', 'none', 'nonnull', 'norecurse', 'notail',
+                'preserve_allcc', 'preserve_mostcc', 'prologue', 'safestack', 'samesize',
+                'source_filename', 'swiftcc', 'swifterror', 'swiftself', 'webkit_jscc',
+                'within', 'writeonly', 'x86_intrcc', 'x86_vectorcallcc'),
+                suffix=r'\b'), Keyword),
+
+            # Types
+            (words(('void', 'half', 'float', 'double', 'x86_fp80', 'fp128',
+                    'ppc_fp128', 'label', 'metadata', 'token')), Keyword.Type),
+
+            # Integer types
+            (r'i[1-9]\d*', Keyword)
+        ]
+    }
+
+
+class NasmLexer(RegexLexer):
+    """
+    For Nasm (Intel) assembly code.
+    """
+    name = 'NASM'
+    aliases = ['nasm']
+    filenames = ['*.asm', '*.ASM']
+    mimetypes = ['text/x-nasm']
+
+    identifier = r'[a-z$._?][\w$.?#@~]*'
+    hexn = r'(?:0x[0-9a-f]+|$0[0-9a-f]*|[0-9]+[0-9a-f]*h)'
+    octn = r'[0-7]+q'
+    binn = r'[01]+b'
+    decn = r'[0-9]+'
+    floatn = decn + r'\.e?' + decn
+    string = r'"(\\"|[^"\n])*"|' + r"'(\\'|[^'\n])*'|" + r"`(\\`|[^`\n])*`"
+    declkw = r'(?:res|d)[bwdqt]|times'
+    register = (r'r[0-9][0-5]?[bwd]|'
+                r'[a-d][lh]|[er]?[a-d]x|[er]?[sb]p|[er]?[sd]i|[c-gs]s|st[0-7]|'
+                r'mm[0-7]|cr[0-4]|dr[0-367]|tr[3-7]')
+    wordop = r'seg|wrt|strict'
+    type = r'byte|[dq]?word'
+    directives = (r'BITS|USE16|USE32|SECTION|SEGMENT|ABSOLUTE|EXTERN|GLOBAL|'
+                  r'ORG|ALIGN|STRUC|ENDSTRUC|COMMON|CPU|GROUP|UPPERCASE|IMPORT|'
+                  r'EXPORT|LIBRARY|MODULE')
+
+    flags = re.IGNORECASE | re.MULTILINE
+    tokens = {
+        'root': [
+            (r'^\s*%', Comment.Preproc, 'preproc'),
+            include('whitespace'),
+            (identifier + ':', Name.Label),
+            (r'(%s)(\s+)(equ)' % identifier,
+                bygroups(Name.Constant, Keyword.Declaration, Keyword.Declaration),
+                'instruction-args'),
+            (directives, Keyword, 'instruction-args'),
+            (declkw, Keyword.Declaration, 'instruction-args'),
+            (identifier, Name.Function, 'instruction-args'),
+            (r'[\r\n]+', Text)
+        ],
+        'instruction-args': [
+            (string, String),
+            (hexn, Number.Hex),
+            (octn, Number.Oct),
+            (binn, Number.Bin),
+            (floatn, Number.Float),
+            (decn, Number.Integer),
+            include('punctuation'),
+            (register, Name.Builtin),
+            (identifier, Name.Variable),
+            (r'[\r\n]+', Text, '#pop'),
+            include('whitespace')
+        ],
+        'preproc': [
+            (r'[^;\n]+', Comment.Preproc),
+            (r';.*?\n', Comment.Single, '#pop'),
+            (r'\n', Comment.Preproc, '#pop'),
+        ],
+        'whitespace': [
+            (r'\n', Text),
+            (r'[ \t]+', Text),
+            (r';.*', Comment.Single)
+        ],
+        'punctuation': [
+            (r'[,():\[\]]+', Punctuation),
+            (r'[&|^<>+*/%~-]+', Operator),
+            (r'[$]+', Keyword.Constant),
+            (wordop, Operator.Word),
+            (type, Keyword.Type)
+        ],
+    }
+
+
+class NasmObjdumpLexer(ObjdumpLexer):
+    """
+    For the output of 'objdump -d -M intel'.
+
+    .. versionadded:: 2.0
+    """
+    name = 'objdump-nasm'
+    aliases = ['objdump-nasm']
+    filenames = ['*.objdump-intel']
+    mimetypes = ['text/x-nasm-objdump']
+
+    tokens = _objdump_lexer_tokens(NasmLexer)
+
+
+class TasmLexer(RegexLexer):
+    """
+    For Tasm (Turbo Assembler) assembly code.
+    """
+    name = 'TASM'
+    aliases = ['tasm']
+    filenames = ['*.asm', '*.ASM', '*.tasm']
+    mimetypes = ['text/x-tasm']
+
+    identifier = r'[@a-z$._?][\w$.?#@~]*'
+    hexn = r'(?:0x[0-9a-f]+|$0[0-9a-f]*|[0-9]+[0-9a-f]*h)'
+    octn = r'[0-7]+q'
+    binn = r'[01]+b'
+    decn = r'[0-9]+'
+    floatn = decn + r'\.e?' + decn
+    string = r'"(\\"|[^"\n])*"|' + r"'(\\'|[^'\n])*'|" + r"`(\\`|[^`\n])*`"
+    declkw = r'(?:res|d)[bwdqt]|times'
+    register = (r'r[0-9][0-5]?[bwd]|'
+                r'[a-d][lh]|[er]?[a-d]x|[er]?[sb]p|[er]?[sd]i|[c-gs]s|st[0-7]|'
+                r'mm[0-7]|cr[0-4]|dr[0-367]|tr[3-7]')
+    wordop = r'seg|wrt|strict'
+    type = r'byte|[dq]?word'
+    directives = (r'BITS|USE16|USE32|SECTION|SEGMENT|ABSOLUTE|EXTERN|GLOBAL|'
+                  r'ORG|ALIGN|STRUC|ENDSTRUC|ENDS|COMMON|CPU|GROUP|UPPERCASE|INCLUDE|'
+                  r'EXPORT|LIBRARY|MODULE|PROC|ENDP|USES|ARG|DATASEG|UDATASEG|END|IDEAL|'
+                  r'P386|MODEL|ASSUME|CODESEG|SIZE')
+    # T[A-Z][a-z] is more of a convention. Lexer should filter out STRUC definitions
+    # and then 'add' them to datatype somehow.
+    datatype = (r'db|dd|dw|T[A-Z][a-z]+')
+
+    flags = re.IGNORECASE | re.MULTILINE
+    tokens = {
+        'root': [
+            (r'^\s*%', Comment.Preproc, 'preproc'),
+            include('whitespace'),
+            (identifier + ':', Name.Label),
+            (directives, Keyword, 'instruction-args'),
+            (r'(%s)(\s+)(%s)' % (identifier, datatype),
+                bygroups(Name.Constant, Keyword.Declaration, Keyword.Declaration),
+                'instruction-args'),
+            (declkw, Keyword.Declaration, 'instruction-args'),
+            (identifier, Name.Function, 'instruction-args'),
+            (r'[\r\n]+', Text)
+        ],
+        'instruction-args': [
+            (string, String),
+            (hexn, Number.Hex),
+            (octn, Number.Oct),
+            (binn, Number.Bin),
+            (floatn, Number.Float),
+            (decn, Number.Integer),
+            include('punctuation'),
+            (register, Name.Builtin),
+            (identifier, Name.Variable),
+            # Do not match newline when it's preceeded by a backslash
+            (r'(\\\s*)(;.*)([\r\n])', bygroups(Text, Comment.Single, Text)),
+            (r'[\r\n]+', Text, '#pop'),
+            include('whitespace')
+        ],
+        'preproc': [
+            (r'[^;\n]+', Comment.Preproc),
+            (r';.*?\n', Comment.Single, '#pop'),
+            (r'\n', Comment.Preproc, '#pop'),
+        ],
+        'whitespace': [
+            (r'[\n\r]', Text),
+            (r'\\[\n\r]', Text),
+            (r'[ \t]+', Text),
+            (r';.*', Comment.Single)
+        ],
+        'punctuation': [
+            (r'[,():\[\]]+', Punctuation),
+            (r'[&|^<>+*=/%~-]+', Operator),
+            (r'[$]+', Keyword.Constant),
+            (wordop, Operator.Word),
+            (type, Keyword.Type)
+        ],
+    }
+
+
+class Ca65Lexer(RegexLexer):
+    """
+    For ca65 assembler sources.
+
+    .. versionadded:: 1.6
+    """
+    name = 'ca65 assembler'
+    aliases = ['ca65']
+    filenames = ['*.s']
+
+    flags = re.IGNORECASE
+
+    tokens = {
+        'root': [
+            (r';.*', Comment.Single),
+            (r'\s+', Text),
+            (r'[a-z_.@$][\w.@$]*:', Name.Label),
+            (r'((ld|st)[axy]|(in|de)[cxy]|asl|lsr|ro[lr]|adc|sbc|cmp|cp[xy]'
+             r'|cl[cvdi]|se[cdi]|jmp|jsr|bne|beq|bpl|bmi|bvc|bvs|bcc|bcs'
+             r'|p[lh][ap]|rt[is]|brk|nop|ta[xy]|t[xy]a|txs|tsx|and|ora|eor'
+             r'|bit)\b', Keyword),
+            (r'\.\w+', Keyword.Pseudo),
+            (r'[-+~*/^&|!<>=]', Operator),
+            (r'"[^"\n]*.', String),
+            (r"'[^'\n]*.", String.Char),
+            (r'\$[0-9a-f]+|[0-9a-f]+h\b', Number.Hex),
+            (r'\d+', Number.Integer),
+            (r'%[01]+', Number.Bin),
+            (r'[#,.:()=\[\]]', Punctuation),
+            (r'[a-z_.@$][\w.@$]*', Name),
+        ]
+    }
+
+    def analyse_text(self, text):
+        # comments in GAS start with "#"
+        if re.match(r'^\s*;', text, re.MULTILINE):
+            return 0.9

packages/wakatime/packages/pygments/lexers/automation.py

@@ -5,7 +5,7 @@
 
     Lexers for automation scripting languages.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -31,8 +31,8 @@ class AutohotkeyLexer(RegexLexer):
         'root': [
             (r'^(\s*)(/\*)', bygroups(Text, Comment.Multiline), 'incomment'),
             (r'^(\s*)(\()', bygroups(Text, Generic), 'incontinuation'),
-            (r'\s+;.*?$', Comment.Singleline),
-            (r'^;.*?$', Comment.Singleline),
+            (r'\s+;.*?$', Comment.Single),
+            (r'^;.*?$', Comment.Single),
             (r'[]{}(),;[]', Punctuation),
             (r'(in|is|and|or|not)\b', Operator.Word),
             (r'\%[a-zA-Z_#@$][\w#@$]*\%', Name.Variable),
@@ -316,7 +316,8 @@ class AutoItLexer(RegexLexer):
     tokens = {
         'root': [
             (r';.*\n', Comment.Single),
-            (r'(#comments-start|#cs).*?(#comments-end|#ce)', Comment.Multiline),
+            (r'(#comments-start|#cs)(.|\n)*?(#comments-end|#ce)',
+             Comment.Multiline),
             (r'[\[\]{}(),;]', Punctuation),
             (r'(and|or|not)\b', Operator.Word),
             (r'[$|@][a-zA-Z_]\w*', Name.Variable),

packages/wakatime/packages/pygments/lexers/basic.py

@@ -5,7 +5,7 @@
 
     Lexers for BASIC like languages (other than VB.net).
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/bibtex.py

@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.bibtex
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for BibTeX bibliography data and styles
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, ExtendedRegexLexer, include, default, \
+    words
+from pygments.token import Name, Comment, String, Error, Number, Text, \
+    Keyword, Punctuation
+
+__all__ = ['BibTeXLexer', 'BSTLexer']
+
+
+class BibTeXLexer(ExtendedRegexLexer):
+    """
+    A lexer for BibTeX bibliography data format.
+
+    .. versionadded:: 2.2
+    """
+
+    name = 'BibTeX'
+    aliases = ['bib', 'bibtex']
+    filenames = ['*.bib']
+    mimetypes = ["text/x-bibtex"]
+    flags = re.IGNORECASE
+
+    ALLOWED_CHARS = r'@!$&*+\-./:;<>?\[\\\]^`|~'
+    IDENTIFIER = '[{0}][{1}]*'.format('a-z_' + ALLOWED_CHARS, r'\w' + ALLOWED_CHARS)
+
+    def open_brace_callback(self, match, ctx):
+        opening_brace = match.group()
+        ctx.opening_brace = opening_brace
+        yield match.start(), Punctuation, opening_brace
+        ctx.pos = match.end()
+
+    def close_brace_callback(self, match, ctx):
+        closing_brace = match.group()
+        if (
+            ctx.opening_brace == '{' and closing_brace != '}' or
+            ctx.opening_brace == '(' and closing_brace != ')'
+        ):
+            yield match.start(), Error, closing_brace
+        else:
+            yield match.start(), Punctuation, closing_brace
+        del ctx.opening_brace
+        ctx.pos = match.end()
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            ('@comment', Comment),
+            ('@preamble', Name.Class, ('closing-brace', 'value', 'opening-brace')),
+            ('@string', Name.Class, ('closing-brace', 'field', 'opening-brace')),
+            ('@' + IDENTIFIER, Name.Class,
+             ('closing-brace', 'command-body', 'opening-brace')),
+            ('.+', Comment),
+        ],
+        'opening-brace': [
+            include('whitespace'),
+            (r'[{(]', open_brace_callback, '#pop'),
+        ],
+        'closing-brace': [
+            include('whitespace'),
+            (r'[})]', close_brace_callback, '#pop'),
+        ],
+        'command-body': [
+            include('whitespace'),
+            (r'[^\s\,\}]+', Name.Label, ('#pop', 'fields')),
+        ],
+        'fields': [
+            include('whitespace'),
+            (',', Punctuation, 'field'),
+            default('#pop'),
+        ],
+        'field': [
+            include('whitespace'),
+            (IDENTIFIER, Name.Attribute, ('value', '=')),
+            default('#pop'),
+        ],
+        '=': [
+            include('whitespace'),
+            ('=', Punctuation, '#pop'),
+        ],
+        'value': [
+            include('whitespace'),
+            (IDENTIFIER, Name.Variable),
+            ('"', String, 'quoted-string'),
+            (r'\{', String, 'braced-string'),
+            (r'[\d]+', Number),
+            ('#', Punctuation),
+            default('#pop'),
+        ],
+        'quoted-string': [
+            (r'\{', String, 'braced-string'),
+            ('"', String, '#pop'),
+            ('[^\{\"]+', String),
+        ],
+        'braced-string': [
+            (r'\{', String, '#push'),
+            (r'\}', String, '#pop'),
+            ('[^\{\}]+', String),
+        ],
+        'whitespace': [
+            (r'\s+', Text),
+        ],
+    }
+
+
+class BSTLexer(RegexLexer):
+    """
+    A lexer for BibTeX bibliography styles.
+
+    .. versionadded:: 2.2
+    """
+
+    name = 'BST'
+    aliases = ['bst', 'bst-pybtex']
+    filenames = ['*.bst']
+    flags = re.IGNORECASE | re.MULTILINE
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            (words(['read', 'sort']), Keyword),
+            (words(['execute', 'integers', 'iterate', 'reverse', 'strings']),
+             Keyword, ('group')),
+            (words(['function', 'macro']), Keyword, ('group', 'group')),
+            (words(['entry']), Keyword, ('group', 'group', 'group')),
+        ],
+        'group': [
+            include('whitespace'),
+            (r'\{', Punctuation, ('#pop', 'group-end', 'body')),
+        ],
+        'group-end': [
+            include('whitespace'),
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'body': [
+            include('whitespace'),
+            (r"\'[^#\"\{\}\s]+", Name.Function),
+            (r'[^#\"\{\}\s]+\$', Name.Builtin),
+            (r'[^#\"\{\}\s]+', Name.Variable),
+            (r'"[^\"]*"', String),
+            (r'#-?\d+', Number),
+            (r'\{', Punctuation, ('group-end', 'body')),
+            default('#pop'),
+        ],
+        'whitespace': [
+            ('\s+', Text),
+            ('%.*?$', Comment.SingleLine),
+        ],
+    }

packages/wakatime/packages/pygments/lexers/business.py

@@ -5,7 +5,7 @@
 
     Lexers for "business-oriented" languages.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -57,9 +57,9 @@ class CobolLexer(RegexLexer):
         ],
         'core': [
             # Figurative constants
-            (r'(^|(?<=[^0-9a-z_\-]))(ALL\s+)?'
+            (r'(^|(?<=[^\w\-]))(ALL\s+)?'
              r'((ZEROES)|(HIGH-VALUE|LOW-VALUE|QUOTE|SPACE|ZERO)(S)?)'
-             r'\s*($|(?=[^0-9a-z_\-]))',
+             r'\s*($|(?=[^\w\-]))',
              Name.Constant),
 
             # Reserved words STATEMENTS and other bolds
@@ -79,8 +79,8 @@ class CobolLexer(RegexLexer):
                 'RETURN', 'REWRITE', 'SCREEN', 'SD', 'SEARCH', 'SECTION', 'SET',
                 'SORT', 'START', 'STOP', 'STRING', 'SUBTRACT', 'SUPPRESS',
                 'TERMINATE', 'THEN', 'UNLOCK', 'UNSTRING', 'USE', 'VALIDATE',
-                'WORKING-STORAGE', 'WRITE'), prefix=r'(^|(?<=[^0-9a-z_\-]))',
-                suffix=r'\s*($|(?=[^0-9a-z_\-]))'),
+                'WORKING-STORAGE', 'WRITE'), prefix=r'(^|(?<=[^\w\-]))',
+                suffix=r'\s*($|(?=[^\w\-]))'),
              Keyword.Reserved),
 
             # Reserved words
@@ -89,33 +89,33 @@ class CobolLexer(RegexLexer):
                 'ALPHABET', 'ALPHABETIC', 'ALPHABETIC-LOWER', 'ALPHABETIC-UPPER',
                 'ALPHANUMERIC', 'ALPHANUMERIC-EDITED', 'ALSO', 'ALTER', 'ALTERNATE'
                 'ANY', 'ARE', 'AREA', 'AREAS', 'ARGUMENT-NUMBER', 'ARGUMENT-VALUE', 'AS',
-                'ASCENDING', 'ASSIGN', 'AT', 'AUTO', 'AUTO-SKIP', 'AUTOMATIC', 'AUTOTERMINATE',
-                'BACKGROUND-COLOR', 'BASED', 'BEEP', 'BEFORE', 'BELL',
+                'ASCENDING', 'ASSIGN', 'AT', 'AUTO', 'AUTO-SKIP', 'AUTOMATIC',
+                'AUTOTERMINATE', 'BACKGROUND-COLOR', 'BASED', 'BEEP', 'BEFORE', 'BELL',
                 'BLANK', 'BLINK', 'BLOCK', 'BOTTOM', 'BY', 'BYTE-LENGTH', 'CHAINING',
-                'CHARACTER', 'CHARACTERS', 'CLASS', 'CODE', 'CODE-SET', 'COL', 'COLLATING',
-                'COLS', 'COLUMN', 'COLUMNS', 'COMMA', 'COMMAND-LINE', 'COMMIT', 'COMMON',
-                'CONSTANT', 'CONTAINS', 'CONTENT', 'CONTROL',
+                'CHARACTER', 'CHARACTERS', 'CLASS', 'CODE', 'CODE-SET', 'COL',
+                'COLLATING', 'COLS', 'COLUMN', 'COLUMNS', 'COMMA', 'COMMAND-LINE',
+                'COMMIT', 'COMMON', 'CONSTANT', 'CONTAINS', 'CONTENT', 'CONTROL',
                 'CONTROLS', 'CONVERTING', 'COPY', 'CORR', 'CORRESPONDING', 'COUNT', 'CRT',
-                'CURRENCY', 'CURSOR', 'CYCLE', 'DATE', 'DAY', 'DAY-OF-WEEK', 'DE', 'DEBUGGING',
-                'DECIMAL-POINT', 'DECLARATIVES', 'DEFAULT', 'DELIMITED',
+                'CURRENCY', 'CURSOR', 'CYCLE', 'DATE', 'DAY', 'DAY-OF-WEEK', 'DE',
+                'DEBUGGING', 'DECIMAL-POINT', 'DECLARATIVES', 'DEFAULT', 'DELIMITED',
                 'DELIMITER', 'DEPENDING', 'DESCENDING', 'DETAIL', 'DISK',
                 'DOWN', 'DUPLICATES', 'DYNAMIC', 'EBCDIC',
                 'ENTRY', 'ENVIRONMENT-NAME', 'ENVIRONMENT-VALUE', 'EOL', 'EOP',
                 'EOS', 'ERASE', 'ERROR', 'ESCAPE', 'EXCEPTION',
-                'EXCLUSIVE', 'EXTEND', 'EXTERNAL',
-                'FILE-ID', 'FILLER', 'FINAL', 'FIRST', 'FIXED', 'FLOAT-LONG', 'FLOAT-SHORT',
-                'FOOTING', 'FOR', 'FOREGROUND-COLOR', 'FORMAT', 'FROM', 'FULL', 'FUNCTION',
-                'FUNCTION-ID', 'GIVING', 'GLOBAL', 'GROUP',
+                'EXCLUSIVE', 'EXTEND', 'EXTERNAL', 'FILE-ID', 'FILLER', 'FINAL',
+                'FIRST', 'FIXED', 'FLOAT-LONG', 'FLOAT-SHORT',
+                'FOOTING', 'FOR', 'FOREGROUND-COLOR', 'FORMAT', 'FROM', 'FULL',
+                'FUNCTION', 'FUNCTION-ID', 'GIVING', 'GLOBAL', 'GROUP',
                 'HEADING', 'HIGHLIGHT', 'I-O', 'ID',
                 'IGNORE', 'IGNORING', 'IN', 'INDEX', 'INDEXED', 'INDICATE',
-                'INITIAL', 'INITIALIZED', 'INPUT',
-                'INTO', 'INTRINSIC', 'INVALID', 'IS', 'JUST', 'JUSTIFIED', 'KEY', 'LABEL',
+                'INITIAL', 'INITIALIZED', 'INPUT', 'INTO', 'INTRINSIC', 'INVALID',
+                'IS', 'JUST', 'JUSTIFIED', 'KEY', 'LABEL',
                 'LAST', 'LEADING', 'LEFT', 'LENGTH', 'LIMIT', 'LIMITS', 'LINAGE',
                 'LINAGE-COUNTER', 'LINE', 'LINES', 'LOCALE', 'LOCK',
-                'LOWLIGHT', 'MANUAL', 'MEMORY', 'MINUS', 'MODE',
-                'MULTIPLE', 'NATIONAL', 'NATIONAL-EDITED', 'NATIVE',
-                'NEGATIVE', 'NEXT', 'NO', 'NULL', 'NULLS', 'NUMBER', 'NUMBERS', 'NUMERIC',
-                'NUMERIC-EDITED', 'OBJECT-COMPUTER', 'OCCURS', 'OF', 'OFF', 'OMITTED', 'ON', 'ONLY',
+                'LOWLIGHT', 'MANUAL', 'MEMORY', 'MINUS', 'MODE', 'MULTIPLE',
+                'NATIONAL', 'NATIONAL-EDITED', 'NATIVE', 'NEGATIVE', 'NEXT', 'NO',
+                'NULL', 'NULLS', 'NUMBER', 'NUMBERS', 'NUMERIC', 'NUMERIC-EDITED',
+                'OBJECT-COMPUTER', 'OCCURS', 'OF', 'OFF', 'OMITTED', 'ON', 'ONLY',
                 'OPTIONAL', 'ORDER', 'ORGANIZATION', 'OTHER', 'OUTPUT', 'OVERFLOW',
                 'OVERLINE', 'PACKED-DECIMAL', 'PADDING', 'PAGE', 'PARAGRAPH',
                 'PLUS', 'POINTER', 'POSITION', 'POSITIVE', 'PRESENT', 'PREVIOUS',
@@ -137,40 +137,42 @@ class CobolLexer(RegexLexer):
                 'UNSIGNED-INT', 'UNSIGNED-LONG', 'UNSIGNED-SHORT', 'UNTIL', 'UP',
                 'UPDATE', 'UPON', 'USAGE', 'USING', 'VALUE', 'VALUES', 'VARYING',
                 'WAIT', 'WHEN', 'WITH', 'WORDS', 'YYYYDDD', 'YYYYMMDD'),
-                prefix=r'(^|(?<=[^0-9a-z_\-]))', suffix=r'\s*($|(?=[^0-9a-z_\-]))'),
+                prefix=r'(^|(?<=[^\w\-]))', suffix=r'\s*($|(?=[^\w\-]))'),
              Keyword.Pseudo),
 
             # inactive reserved words
             (words((
-                'ACTIVE-CLASS', 'ALIGNED', 'ANYCASE', 'ARITHMETIC', 'ATTRIBUTE', 'B-AND',
-                'B-NOT', 'B-OR', 'B-XOR', 'BIT', 'BOOLEAN', 'CD', 'CENTER', 'CF', 'CH', 'CHAIN', 'CLASS-ID',
-                'CLASSIFICATION', 'COMMUNICATION', 'CONDITION', 'DATA-POINTER',
-                'DESTINATION', 'DISABLE', 'EC', 'EGI', 'EMI', 'ENABLE', 'END-RECEIVE',
-                'ENTRY-CONVENTION', 'EO', 'ESI', 'EXCEPTION-OBJECT', 'EXPANDS', 'FACTORY',
-                'FLOAT-BINARY-16', 'FLOAT-BINARY-34', 'FLOAT-BINARY-7',
-                'FLOAT-DECIMAL-16', 'FLOAT-DECIMAL-34', 'FLOAT-EXTENDED', 'FORMAT',
-                'FUNCTION-POINTER', 'GET', 'GROUP-USAGE', 'IMPLEMENTS', 'INFINITY',
-                'INHERITS', 'INTERFACE', 'INTERFACE-ID', 'INVOKE', 'LC_ALL', 'LC_COLLATE',
+                'ACTIVE-CLASS', 'ALIGNED', 'ANYCASE', 'ARITHMETIC', 'ATTRIBUTE',
+                'B-AND', 'B-NOT', 'B-OR', 'B-XOR', 'BIT', 'BOOLEAN', 'CD', 'CENTER',
+                'CF', 'CH', 'CHAIN', 'CLASS-ID', 'CLASSIFICATION', 'COMMUNICATION',
+                'CONDITION', 'DATA-POINTER', 'DESTINATION', 'DISABLE', 'EC', 'EGI',
+                'EMI', 'ENABLE', 'END-RECEIVE', 'ENTRY-CONVENTION', 'EO', 'ESI',
+                'EXCEPTION-OBJECT', 'EXPANDS', 'FACTORY', 'FLOAT-BINARY-16',
+                'FLOAT-BINARY-34', 'FLOAT-BINARY-7', 'FLOAT-DECIMAL-16',
+                'FLOAT-DECIMAL-34', 'FLOAT-EXTENDED', 'FORMAT', 'FUNCTION-POINTER',
+                'GET', 'GROUP-USAGE', 'IMPLEMENTS', 'INFINITY', 'INHERITS',
+                'INTERFACE', 'INTERFACE-ID', 'INVOKE', 'LC_ALL', 'LC_COLLATE',
                 'LC_CTYPE', 'LC_MESSAGES', 'LC_MONETARY', 'LC_NUMERIC', 'LC_TIME',
-                'LINE-COUNTER', 'MESSAGE', 'METHOD', 'METHOD-ID', 'NESTED', 'NONE', 'NORMAL',
-                'OBJECT', 'OBJECT-REFERENCE', 'OPTIONS', 'OVERRIDE', 'PAGE-COUNTER', 'PF', 'PH',
-                'PROPERTY', 'PROTOTYPE', 'PURGE', 'QUEUE', 'RAISE', 'RAISING', 'RECEIVE',
-                'RELATION', 'REPLACE', 'REPRESENTS-NOT-A-NUMBER', 'RESET', 'RESUME', 'RETRY',
-                'RF', 'RH', 'SECONDS', 'SEGMENT', 'SELF', 'SEND', 'SOURCES', 'STATEMENT', 'STEP',
-                'STRONG', 'SUB-QUEUE-1', 'SUB-QUEUE-2', 'SUB-QUEUE-3', 'SUPER', 'SYMBOL',
-                'SYSTEM-DEFAULT', 'TABLE', 'TERMINAL', 'TEXT', 'TYPEDEF', 'UCS-4', 'UNIVERSAL',
-                'USER-DEFAULT', 'UTF-16', 'UTF-8', 'VAL-STATUS', 'VALID', 'VALIDATE',
-                'VALIDATE-STATUS'),
-                prefix=r'(^|(?<=[^0-9a-z_\-]))', suffix=r'\s*($|(?=[^0-9a-z_\-]))'),
+                'LINE-COUNTER', 'MESSAGE', 'METHOD', 'METHOD-ID', 'NESTED', 'NONE',
+                'NORMAL', 'OBJECT', 'OBJECT-REFERENCE', 'OPTIONS', 'OVERRIDE',
+                'PAGE-COUNTER', 'PF', 'PH', 'PROPERTY', 'PROTOTYPE', 'PURGE',
+                'QUEUE', 'RAISE', 'RAISING', 'RECEIVE', 'RELATION', 'REPLACE',
+                'REPRESENTS-NOT-A-NUMBER', 'RESET', 'RESUME', 'RETRY', 'RF', 'RH',
+                'SECONDS', 'SEGMENT', 'SELF', 'SEND', 'SOURCES', 'STATEMENT',
+                'STEP', 'STRONG', 'SUB-QUEUE-1', 'SUB-QUEUE-2', 'SUB-QUEUE-3',
+                'SUPER', 'SYMBOL', 'SYSTEM-DEFAULT', 'TABLE', 'TERMINAL', 'TEXT',
+                'TYPEDEF', 'UCS-4', 'UNIVERSAL', 'USER-DEFAULT', 'UTF-16', 'UTF-8',
+                'VAL-STATUS', 'VALID', 'VALIDATE', 'VALIDATE-STATUS'),
+                   prefix=r'(^|(?<=[^\w\-]))', suffix=r'\s*($|(?=[^\w\-]))'),
              Error),
 
             # Data Types
-            (r'(^|(?<=[^0-9a-z_\-]))'
+            (r'(^|(?<=[^\w\-]))'
              r'(PIC\s+.+?(?=(\s|\.\s))|PICTURE\s+.+?(?=(\s|\.\s))|'
              r'(COMPUTATIONAL)(-[1-5X])?|(COMP)(-[1-5X])?|'
              r'BINARY-C-LONG|'
              r'BINARY-CHAR|BINARY-DOUBLE|BINARY-LONG|BINARY-SHORT|'
-             r'BINARY)\s*($|(?=[^0-9a-z_\-]))', Keyword.Type),
+             r'BINARY)\s*($|(?=[^\w\-]))', Keyword.Type),
 
             # Operators
             (r'(\*\*|\*|\+|-|/|<=|>=|<|>|==|/=|=)', Operator),
@@ -180,7 +182,7 @@ class CobolLexer(RegexLexer):
             (r'([(),;:&%.])', Punctuation),
 
             # Intrinsics
-            (r'(^|(?<=[^0-9a-z_\-]))(ABS|ACOS|ANNUITY|ASIN|ATAN|BYTE-LENGTH|'
+            (r'(^|(?<=[^\w\-]))(ABS|ACOS|ANNUITY|ASIN|ATAN|BYTE-LENGTH|'
              r'CHAR|COMBINED-DATETIME|CONCATENATE|COS|CURRENT-DATE|'
              r'DATE-OF-INTEGER|DATE-TO-YYYYMMDD|DAY-OF-INTEGER|DAY-TO-YYYYDDD|'
              r'EXCEPTION-(?:FILE|LOCATION|STATEMENT|STATUS)|EXP10|EXP|E|'
@@ -192,13 +194,13 @@ class CobolLexer(RegexLexer):
              r'STANDARD-DEVIATION|STORED-CHAR-LENGTH|SUBSTITUTE(?:-CASE)?|'
              r'SUM|TAN|TEST-DATE-YYYYMMDD|TEST-DAY-YYYYDDD|TRIM|'
              r'UPPER-CASE|VARIANCE|WHEN-COMPILED|YEAR-TO-YYYY)\s*'
-             r'($|(?=[^0-9a-z_\-]))', Name.Function),
+             r'($|(?=[^\w\-]))', Name.Function),
 
             # Booleans
-            (r'(^|(?<=[^0-9a-z_\-]))(true|false)\s*($|(?=[^0-9a-z_\-]))', Name.Builtin),
+            (r'(^|(?<=[^\w\-]))(true|false)\s*($|(?=[^\w\-]))', Name.Builtin),
             # Comparing Operators
-            (r'(^|(?<=[^0-9a-z_\-]))(equal|equals|ne|lt|le|gt|ge|'
-             r'greater|less|than|not|and|or)\s*($|(?=[^0-9a-z_\-]))', Operator.Word),
+            (r'(^|(?<=[^\w\-]))(equal|equals|ne|lt|le|gt|ge|'
+             r'greater|less|than|not|and|or)\s*($|(?=[^\w\-]))', Operator.Word),
         ],
 
         # \"[^\"\n]*\"|\'[^\'\n]*\'
@@ -244,7 +246,7 @@ class ABAPLexer(RegexLexer):
     """
     name = 'ABAP'
     aliases = ['abap']
-    filenames = ['*.abap']
+    filenames = ['*.abap', '*.ABAP']
     mimetypes = ['text/x-abap']
 
     flags = re.IGNORECASE | re.MULTILINE
@@ -254,6 +256,7 @@ class ABAPLexer(RegexLexer):
             (r'\s+', Text),
             (r'^\*.*$', Comment.Single),
             (r'\".*?\n', Comment.Single),
+            (r'##\w+', Comment.Special),
         ],
         'variable-names': [
             (r'<\S+>', Name.Variable),
@@ -262,8 +265,8 @@ class ABAPLexer(RegexLexer):
         'root': [
             include('common'),
             # function calls
-            (r'(CALL\s+(?:BADI|CUSTOMER-FUNCTION|FUNCTION))(\s+)(\'?\S+\'?)',
-             bygroups(Keyword, Text, Name.Function)),
+            (r'CALL\s+(?:BADI|CUSTOMER-FUNCTION|FUNCTION)',
+             Keyword),
             (r'(CALL\s+(?:DIALOG|SCREEN|SUBSCREEN|SELECTION-SCREEN|'
              r'TRANSACTION|TRANSFORMATION))\b',
              Keyword),
@@ -283,6 +286,12 @@ class ABAPLexer(RegexLexer):
             # call methodnames returning style
             (r'(?<=(=|-)>)([\w\-~]+)(?=\()', Name.Function),
 
+            # text elements
+            (r'(TEXT)(-)(\d{3})',
+             bygroups(Keyword, Punctuation, Number.Integer)),
+            (r'(TEXT)(-)(\w{3})',
+             bygroups(Keyword, Punctuation, Name.Variable)),
+
             # keywords with dashes in them.
             # these need to be first, because for instance the -ID part
             # of MESSAGE-ID wouldn't get highlighted if MESSAGE was
@@ -299,13 +308,13 @@ class ABAPLexer(RegexLexer):
              r'OUTPUT-LENGTH|PRINT-CONTROL|'
              r'SELECT-OPTIONS|START-OF-SELECTION|SUBTRACT-CORRESPONDING|'
              r'SYNTAX-CHECK|SYSTEM-EXCEPTIONS|'
-             r'TYPE-POOL|TYPE-POOLS'
+             r'TYPE-POOL|TYPE-POOLS|NO-DISPLAY'
              r')\b', Keyword),
 
             # keyword kombinations
-            (r'CREATE\s+(PUBLIC|PRIVATE|DATA|OBJECT)|'
-             r'((PUBLIC|PRIVATE|PROTECTED)\s+SECTION|'
-             r'(TYPE|LIKE)(\s+(LINE\s+OF|REF\s+TO|'
+            (r'(?<![-\>])(CREATE\s+(PUBLIC|PRIVATE|DATA|OBJECT)|'
+             r'(PUBLIC|PRIVATE|PROTECTED)\s+SECTION|'
+             r'(TYPE|LIKE)\s+((LINE\s+OF|REF\s+TO|'
              r'(SORTED|STANDARD|HASHED)\s+TABLE\s+OF))?|'
              r'FROM\s+(DATABASE|MEMORY)|CALL\s+METHOD|'
              r'(GROUP|ORDER) BY|HAVING|SEPARATED BY|'
@@ -341,10 +350,16 @@ class ABAPLexer(RegexLexer):
              r'(BEGIN|END)\s+OF|'
              r'DELETE(\s+ADJACENT\s+DUPLICATES\sFROM)?|'
              r'COMPARING(\s+ALL\s+FIELDS)?|'
-             r'INSERT(\s+INITIAL\s+LINE\s+INTO|\s+LINES\s+OF)?|'
+             r'(INSERT|APPEND)(\s+INITIAL\s+LINE\s+(IN)?TO|\s+LINES\s+OF)?|'
              r'IN\s+((BYTE|CHARACTER)\s+MODE|PROGRAM)|'
              r'END-OF-(DEFINITION|PAGE|SELECTION)|'
              r'WITH\s+FRAME(\s+TITLE)|'
+             r'(REPLACE|FIND)\s+((FIRST|ALL)\s+OCCURRENCES?\s+OF\s+)?(SUBSTRING|REGEX)?|'
+             r'MATCH\s+(LENGTH|COUNT|LINE|OFFSET)|'
+             r'(RESPECTING|IGNORING)\s+CASE|'
+             r'IN\s+UPDATE\s+TASK|'
+             r'(SOURCE|RESULT)\s+(XML)?|'
+             r'REFERENCE\s+INTO|'
 
              # simple kombinations
              r'AND\s+(MARK|RETURN)|CLIENT\s+SPECIFIED|CORRESPONDING\s+FIELDS\s+OF|'
@@ -353,39 +368,41 @@ class ABAPLexer(RegexLexer):
              r'MODIFY\s+SCREEN|NESTING\s+LEVEL|NO\s+INTERVALS|OF\s+STRUCTURE|'
              r'RADIOBUTTON\s+GROUP|RANGE\s+OF|REF\s+TO|SUPPRESS DIALOG|'
              r'TABLE\s+OF|UPPER\s+CASE|TRANSPORTING\s+NO\s+FIELDS|'
-             r'VALUE\s+CHECK|VISIBLE\s+LENGTH|HEADER\s+LINE)\b', Keyword),
+             r'VALUE\s+CHECK|VISIBLE\s+LENGTH|HEADER\s+LINE|COMMON\s+PART)\b', Keyword),
 
             # single word keywords.
-            (r'(^|(?<=(\s|\.)))(ABBREVIATED|ADD|ALIASES|APPEND|ASSERT|'
-             r'ASSIGN(ING)?|AT(\s+FIRST)?|'
+            (r'(^|(?<=(\s|\.)))(ABBREVIATED|ABSTRACT|ADD|ALIASES|ALIGN|ALPHA|'
+             r'ASSERT|AS|ASSIGN(ING)?|AT(\s+FIRST)?|'
              r'BACK|BLOCK|BREAK-POINT|'
              r'CASE|CATCH|CHANGING|CHECK|CLASS|CLEAR|COLLECT|COLOR|COMMIT|'
              r'CREATE|COMMUNICATION|COMPONENTS?|COMPUTE|CONCATENATE|CONDENSE|'
-             r'CONSTANTS|CONTEXTS|CONTINUE|CONTROLS|'
-             r'DATA|DECIMALS|DEFAULT|DEFINE|DEFINITION|DEFERRED|DEMAND|'
-             r'DETAIL|DIRECTORY|DIVIDE|DO|'
-             r'ELSE(IF)?|ENDAT|ENDCASE|ENDCLASS|ENDDO|ENDFORM|ENDFUNCTION|'
-             r'ENDIF|ENDLOOP|ENDMETHOD|ENDMODULE|ENDSELECT|ENDTRY|'
-             r'ENHANCEMENT|EVENTS|EXCEPTIONS|EXIT|EXPORT|EXPORTING|EXTRACT|'
-             r'FETCH|FIELDS?|FIND|FOR|FORM|FORMAT|FREE|FROM|'
+             r'CONSTANTS|CONTEXTS|CONTINUE|CONTROLS|COUNTRY|CURRENCY|'
+             r'DATA|DATE|DECIMALS|DEFAULT|DEFINE|DEFINITION|DEFERRED|DEMAND|'
+             r'DETAIL|DIRECTORY|DIVIDE|DO|DUMMY|'
+             r'ELSE(IF)?|ENDAT|ENDCASE|ENDCATCH|ENDCLASS|ENDDO|ENDFORM|ENDFUNCTION|'
+             r'ENDIF|ENDINTERFACE|ENDLOOP|ENDMETHOD|ENDMODULE|ENDSELECT|ENDTRY|ENDWHILE|'
+             r'ENHANCEMENT|EVENTS|EXACT|EXCEPTIONS?|EXIT|EXPONENT|EXPORT|EXPORTING|EXTRACT|'
+             r'FETCH|FIELDS?|FOR|FORM|FORMAT|FREE|FROM|FUNCTION|'
              r'HIDE|'
              r'ID|IF|IMPORT|IMPLEMENTATION|IMPORTING|IN|INCLUDE|INCLUDING|'
              r'INDEX|INFOTYPES|INITIALIZATION|INTERFACE|INTERFACES|INTO|'
-             r'LENGTH|LINES|LOAD|LOCAL|'
+             r'LANGUAGE|LEAVE|LENGTH|LINES|LOAD|LOCAL|'
              r'JOIN|'
              r'KEY|'
-             r'MAXIMUM|MESSAGE|METHOD[S]?|MINIMUM|MODULE|MODIFY|MOVE|MULTIPLY|'
-             r'NODES|'
-             r'OBLIGATORY|OF|OFF|ON|OVERLAY|'
-             r'PACK|PARAMETERS|PERCENTAGE|POSITION|PROGRAM|PROVIDE|PUBLIC|PUT|'
-             r'RAISE|RAISING|RANGES|READ|RECEIVE|REFRESH|REJECT|REPORT|RESERVE|'
-             r'RESUME|RETRY|RETURN|RETURNING|RIGHT|ROLLBACK|'
-             r'SCROLL|SEARCH|SELECT|SHIFT|SINGLE|SKIP|SORT|SPLIT|STATICS|STOP|'
-             r'SUBMIT|SUBTRACT|SUM|SUMMARY|SUMMING|SUPPLY|'
-             r'TABLE|TABLES|TIMES|TITLE|TO|TOP-OF-PAGE|TRANSFER|TRANSLATE|TRY|TYPES|'
+             r'NEXT|'
+             r'MAXIMUM|MESSAGE|METHOD[S]?|MINIMUM|MODULE|MODIFIER|MODIFY|MOVE|MULTIPLY|'
+             r'NODES|NUMBER|'
+             r'OBLIGATORY|OBJECT|OF|OFF|ON|OTHERS|OVERLAY|'
+             r'PACK|PAD|PARAMETERS|PERCENTAGE|POSITION|PROGRAM|PROVIDE|PUBLIC|PUT|PF\d\d|'
+             r'RAISE|RAISING|RANGES?|READ|RECEIVE|REDEFINITION|REFRESH|REJECT|REPORT|RESERVE|'
+             r'RESUME|RETRY|RETURN|RETURNING|RIGHT|ROLLBACK|REPLACE|'
+             r'SCROLL|SEARCH|SELECT|SHIFT|SIGN|SINGLE|SIZE|SKIP|SORT|SPLIT|STATICS|STOP|'
+             r'STYLE|SUBMATCHES|SUBMIT|SUBTRACT|SUM(?!\()|SUMMARY|SUMMING|SUPPLY|'
+             r'TABLE|TABLES|TIMESTAMP|TIMES?|TIMEZONE|TITLE|\??TO|'
+             r'TOP-OF-PAGE|TRANSFER|TRANSLATE|TRY|TYPES|'
              r'ULINE|UNDER|UNPACK|UPDATE|USING|'
-             r'VALUE|VALUES|VIA|'
-             r'WAIT|WHEN|WHERE|WHILE|WITH|WINDOW|WRITE)\b', Keyword),
+             r'VALUE|VALUES|VIA|VARYING|VARY|'
+             r'WAIT|WHEN|WHERE|WIDTH|WHILE|WITH|WINDOW|WRITE|XSD|ZERO)\b', Keyword),
 
             # builtins
             (r'(abs|acos|asin|atan|'
@@ -411,18 +428,21 @@ class ABAPLexer(RegexLexer):
 
             # operators which look like variable names before
             # parsing variable names.
-            (r'(?<=(\s|.))(AND|EQ|NE|GT|LT|GE|LE|CO|CN|CA|NA|CS|NOT|NS|CP|NP|'
+            (r'(?<=(\s|.))(AND|OR|EQ|NE|GT|LT|GE|LE|CO|CN|CA|NA|CS|NOT|NS|CP|NP|'
              r'BYTE-CO|BYTE-CN|BYTE-CA|BYTE-NA|BYTE-CS|BYTE-NS|'
-             r'IS\s+(NOT\s+)?(INITIAL|ASSIGNED|REQUESTED|BOUND))\b', Operator),
+             r'IS\s+(NOT\s+)?(INITIAL|ASSIGNED|REQUESTED|BOUND))\b', Operator.Word),
 
             include('variable-names'),
 
-            # standard oparators after variable names,
+            # standard operators after variable names,
             # because < and > are part of field symbols.
-            (r'[?*<>=\-+]', Operator),
+            (r'[?*<>=\-+&]', Operator),
             (r"'(''|[^'])*'", String.Single),
             (r"`([^`])*`", String.Single),
-            (r'[/;:()\[\],.]', Punctuation)
+            (r"([|}])([^{}|]*?)([|{])",
+             bygroups(Punctuation, String.Single, Punctuation)),
+            (r'[/;:()\[\],.]', Punctuation),
+            (r'(!)(\w+)', bygroups(Operator, Name)),
         ],
     }
 
@@ -439,15 +459,15 @@ class OpenEdgeLexer(RegexLexer):
     filenames = ['*.p', '*.cls']
     mimetypes = ['text/x-openedge', 'application/x-openedge']
 
-    types = (r'(?i)(^|(?<=[^0-9a-z_\-]))(CHARACTER|CHAR|CHARA|CHARAC|CHARACT|CHARACTE|'
+    types = (r'(?i)(^|(?<=[^\w\-]))(CHARACTER|CHAR|CHARA|CHARAC|CHARACT|CHARACTE|'
              r'COM-HANDLE|DATE|DATETIME|DATETIME-TZ|'
              r'DECIMAL|DEC|DECI|DECIM|DECIMA|HANDLE|'
              r'INT64|INTEGER|INT|INTE|INTEG|INTEGE|'
-             r'LOGICAL|LONGCHAR|MEMPTR|RAW|RECID|ROWID)\s*($|(?=[^0-9a-z_\-]))')
+             r'LOGICAL|LONGCHAR|MEMPTR|RAW|RECID|ROWID)\s*($|(?=[^\w\-]))')
 
     keywords = words(OPENEDGEKEYWORDS,
-                     prefix=r'(?i)(^|(?<=[^0-9a-z_\-]))',
-                     suffix=r'\s*($|(?=[^0-9a-z_\-]))')
+                     prefix=r'(?i)(^|(?<=[^\w\-]))',
+                     suffix=r'\s*($|(?=[^\w\-]))')
 
     tokens = {
         'root': [

packages/wakatime/packages/pygments/lexers/c_cpp.py

@@ -5,7 +5,7 @@
 
     Lexers for C/C++ languages.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -28,8 +28,10 @@ class CFamilyLexer(RegexLexer):
 
     #: optional Comment or Whitespace
     _ws = r'(?:\s|//.*?\n|/[*].*?[*]/)+'
+
+    # The trailing ?, rather than *, avoids a geometric performance drop here.
     #: only one /* */ style comment
-    _ws1 = r'\s*(?:/[*].*?[*]/\s*)*'
+    _ws1 = r'\s*(?:/[*].*?[*]/\s*)?'
 
     tokens = {
         'whitespace': [
@@ -44,12 +46,15 @@ class CFamilyLexer(RegexLexer):
             (r'\n', Text),
             (r'\s+', Text),
             (r'\\\n', Text),  # line continuation
-            (r'//(\n|(.|\n)*?[^\\]\n)', Comment.Single),
-            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+            (r'//(\n|[\w\W]*?[^\\]\n)', Comment.Single),
+            (r'/(\\\n)?[*][\w\W]*?[*](\\\n)?/', Comment.Multiline),
+            # Open until EOF, so no ending delimeter
+            (r'/(\\\n)?[*][\w\W]*', Comment.Multiline),
         ],
         'statements': [
-            (r'L?"', String, 'string'),
-            (r"L?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char),
+            (r'(L?)(")', bygroups(String.Affix, String), 'string'),
+            (r"(L?)(')(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])(')",
+             bygroups(String.Affix, String.Char, String.Char, String.Char)),
             (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*', Number.Float),
             (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
             (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex),
@@ -58,13 +63,13 @@ class CFamilyLexer(RegexLexer):
             (r'\*/', Error),
             (r'[~!%^&*+=|?:<>/-]', Operator),
             (r'[()\[\],.]', Punctuation),
-            (words(('auto', 'break', 'case', 'const', 'continue', 'default', 'do',
-                    'else', 'enum', 'extern', 'for', 'goto', 'if', 'register',
-                    'restricted', 'return', 'sizeof', 'static', 'struct',
-                    'switch', 'typedef', 'union', 'volatile', 'while'),
+            (words(('asm', 'auto', 'break', 'case', 'const', 'continue',
+                    'default', 'do', 'else', 'enum', 'extern', 'for', 'goto',
+                    'if', 'register', 'restricted', 'return', 'sizeof',
+                    'static', 'struct', 'switch', 'typedef', 'union',
+                    'volatile', 'while'),
                    suffix=r'\b'), Keyword),
-            (r'(bool|int|long|float|short|double|char|unsigned|signed|void|'
-             r'[a-z_][a-z0-9_]*_t)\b',
+            (r'(bool|int|long|float|short|double|char|unsigned|signed|void)\b',
              Keyword.Type),
             (words(('inline', '_inline', '__inline', 'naked', 'restrict',
                     'thread', 'typename'), suffix=r'\b'), Keyword.Reserved),
@@ -87,7 +92,7 @@ class CFamilyLexer(RegexLexer):
             (r'((?:[\w*\s])+?(?:\s|[*]))'  # return arguments
              r'([a-zA-Z_]\w*)'             # method name
              r'(\s*\([^;]*?\))'            # signature
-             r'(' + _ws + r')?(\{)',
+             r'([^;{]*)(\{)',
              bygroups(using(this), Name.Function, using(this), using(this),
                       Punctuation),
              'function'),
@@ -95,7 +100,7 @@ class CFamilyLexer(RegexLexer):
             (r'((?:[\w*\s])+?(?:\s|[*]))'  # return arguments
              r'([a-zA-Z_]\w*)'             # method name
              r'(\s*\([^;]*?\))'            # signature
-             r'(' + _ws + r')?(;)',
+             r'([^;]*)(;)',
              bygroups(using(this), Name.Function, using(this), using(this),
                       Punctuation)),
             default('statement'),
@@ -122,6 +127,8 @@ class CFamilyLexer(RegexLexer):
             (r'\\', String),  # stray backslash
         ],
         'macro': [
+            (r'(include)(' + _ws1 + r')([^\n]+)',
+             bygroups(Comment.Preproc, Text, Comment.PreprocFile)),
             (r'[^/\n]+', Comment.Preproc),
             (r'/[*](.|\n)*?[*]/', Comment.Multiline),
             (r'//.*?\n', Comment.Single, '#pop'),
@@ -137,22 +144,26 @@ class CFamilyLexer(RegexLexer):
         ]
     }
 
-    stdlib_types = ['size_t', 'ssize_t', 'off_t', 'wchar_t', 'ptrdiff_t',
-                    'sig_atomic_t', 'fpos_t', 'clock_t', 'time_t', 'va_list',
-                    'jmp_buf', 'FILE', 'DIR', 'div_t', 'ldiv_t', 'mbstate_t',
-                    'wctrans_t', 'wint_t', 'wctype_t']
-    c99_types = ['_Bool', '_Complex', 'int8_t', 'int16_t', 'int32_t', 'int64_t',
-                 'uint8_t', 'uint16_t', 'uint32_t', 'uint64_t', 'int_least8_t',
-                 'int_least16_t', 'int_least32_t', 'int_least64_t',
-                 'uint_least8_t', 'uint_least16_t', 'uint_least32_t',
-                 'uint_least64_t', 'int_fast8_t', 'int_fast16_t', 'int_fast32_t',
-                 'int_fast64_t', 'uint_fast8_t', 'uint_fast16_t', 'uint_fast32_t',
-                 'uint_fast64_t', 'intptr_t', 'uintptr_t', 'intmax_t',
-                 'uintmax_t']
+    stdlib_types = set((
+        'size_t', 'ssize_t', 'off_t', 'wchar_t', 'ptrdiff_t', 'sig_atomic_t', 'fpos_t',
+        'clock_t', 'time_t', 'va_list', 'jmp_buf', 'FILE', 'DIR', 'div_t', 'ldiv_t',
+        'mbstate_t', 'wctrans_t', 'wint_t', 'wctype_t'))
+    c99_types = set((
+        '_Bool', '_Complex', 'int8_t', 'int16_t', 'int32_t', 'int64_t', 'uint8_t',
+        'uint16_t', 'uint32_t', 'uint64_t', 'int_least8_t', 'int_least16_t',
+        'int_least32_t', 'int_least64_t', 'uint_least8_t', 'uint_least16_t',
+        'uint_least32_t', 'uint_least64_t', 'int_fast8_t', 'int_fast16_t', 'int_fast32_t',
+        'int_fast64_t', 'uint_fast8_t', 'uint_fast16_t', 'uint_fast32_t', 'uint_fast64_t',
+        'intptr_t', 'uintptr_t', 'intmax_t', 'uintmax_t'))
+    linux_types = set((
+        'clockid_t', 'cpu_set_t', 'cpumask_t', 'dev_t', 'gid_t', 'id_t', 'ino_t', 'key_t',
+        'mode_t', 'nfds_t', 'pid_t', 'rlim_t', 'sig_t', 'sighandler_t', 'siginfo_t',
+        'sigset_t', 'sigval_t', 'socklen_t', 'timer_t', 'uid_t'))
 
     def __init__(self, **options):
         self.stdlibhighlighting = get_bool_opt(options, 'stdlibhighlighting', True)
         self.c99highlighting = get_bool_opt(options, 'c99highlighting', True)
+        self.platformhighlighting = get_bool_opt(options, 'platformhighlighting', True)
         RegexLexer.__init__(self, **options)
 
     def get_tokens_unprocessed(self, text):
@@ -163,6 +174,8 @@ class CFamilyLexer(RegexLexer):
                     token = Keyword.Type
                 elif self.c99highlighting and value in self.c99_types:
                     token = Keyword.Type
+                elif self.platformhighlighting and value in self.linux_types:
+                    token = Keyword.Type
             yield index, token, value
 
 
@@ -179,7 +192,7 @@ class CLexer(CFamilyLexer):
     def analyse_text(text):
         if re.search('^\s*#include [<"]', text, re.MULTILINE):
             return 0.1
-        if re.search('^\s*#ifdef ', text, re.MULTILINE):
+        if re.search('^\s*#ifn?def ', text, re.MULTILINE):
             return 0.1
 
 
@@ -198,16 +211,22 @@ class CppLexer(CFamilyLexer):
     tokens = {
         'statements': [
             (words((
-                'asm', 'catch', 'const_cast', 'delete', 'dynamic_cast', 'explicit',
+                'catch', 'const_cast', 'delete', 'dynamic_cast', 'explicit',
                 'export', 'friend', 'mutable', 'namespace', 'new', 'operator',
                 'private', 'protected', 'public', 'reinterpret_cast',
                 'restrict', 'static_cast', 'template', 'this', 'throw', 'throws',
-                'typeid', 'typename', 'using', 'virtual',
+                'try', 'typeid', 'typename', 'using', 'virtual',
                 'constexpr', 'nullptr', 'decltype', 'thread_local',
                 'alignas', 'alignof', 'static_assert', 'noexcept', 'override',
                 'final'), suffix=r'\b'), Keyword),
             (r'char(16_t|32_t)\b', Keyword.Type),
             (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'),
+            # C++11 raw strings
+            (r'(R)(")([^\\()\s]{,16})(\()((?:.|\n)*?)(\)\3)(")',
+             bygroups(String.Affix, String, String.Delimiter, String.Delimiter,
+                      String, String.Delimiter, String)),
+            # C++11 UTF-8/16/32 strings
+            (r'(u8|u|U)(")', bygroups(String.Affix, String), 'string'),
             inherit,
         ],
         'root': [
@@ -227,7 +246,7 @@ class CppLexer(CFamilyLexer):
     }
 
     def analyse_text(text):
-        if re.search('#include <[a-z]+>', text):
+        if re.search('#include <[a-z_]+>', text):
             return 0.2
         if re.search('using namespace ', text):
             return 0.4

packages/wakatime/packages/pygments/lexers/c_like.py

@@ -5,7 +5,7 @@
 
     Lexers for other C-like languages.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -20,7 +20,7 @@ from pygments.lexers.c_cpp import CLexer, CppLexer
 from pygments.lexers import _mql_builtins
 
 __all__ = ['PikeLexer', 'NesCLexer', 'ClayLexer', 'ECLexer', 'ValaLexer',
-           'CudaLexer', 'SwigLexer', 'MqlLexer']
+           'CudaLexer', 'SwigLexer', 'MqlLexer', 'ArduinoLexer']
 
 
 class PikeLexer(CppLexer):
@@ -105,7 +105,7 @@ class ClayLexer(RegexLexer):
     tokens = {
         'root': [
             (r'\s', Text),
-            (r'//.*?$', Comment.Singleline),
+            (r'//.*?$', Comment.Single),
             (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
             (r'\b(public|private|import|as|record|variant|instance'
              r'|define|overload|default|external|alias'
@@ -411,3 +411,131 @@ class MqlLexer(CppLexer):
             inherit,
         ],
     }
+
+class ArduinoLexer(CppLexer):
+    """
+    For `Arduino(tm) <https://arduino.cc/>`_ source.
+
+    This is an extension of the CppLexer, as the Arduino® Language is a superset
+    of C++
+
+    .. versionadded:: 2.1
+    """
+
+    name = 'Arduino'
+    aliases = ['arduino']
+    filenames = ['*.ino']
+    mimetypes = ['text/x-arduino']
+
+    # Language sketch main structure functions
+    structure = set(('setup', 'loop'))
+
+    # Language operators
+    operators = set(('not', 'or', 'and', 'xor'))
+
+    # Language 'variables'
+    variables = set((
+        'DIGITAL_MESSAGE', 'FIRMATA_STRING', 'ANALOG_MESSAGE', 'REPORT_DIGITAL',
+        'REPORT_ANALOG', 'INPUT_PULLUP', 'SET_PIN_MODE', 'INTERNAL2V56', 'SYSTEM_RESET',
+        'LED_BUILTIN', 'INTERNAL1V1', 'SYSEX_START', 'INTERNAL', 'EXTERNAL', 'HIGH',
+        'LOW', 'INPUT', 'OUTPUT', 'INPUT_PULLUP', 'LED_BUILTIN', 'true', 'false',
+        'void', 'boolean', 'char', 'unsigned char', 'byte', 'int', 'unsigned int',
+        'word', 'long', 'unsigned long', 'short', 'float', 'double', 'string', 'String',
+        'array', 'static', 'volatile', 'const', 'boolean', 'byte', 'word', 'string',
+        'String', 'array', 'int', 'float', 'private', 'char', 'virtual', 'operator',
+        'sizeof', 'uint8_t', 'uint16_t', 'uint32_t', 'uint64_t', 'int8_t', 'int16_t',
+        'int32_t', 'int64_t', 'dynamic_cast', 'typedef', 'const_cast', 'const',
+        'struct', 'static_cast', 'union', 'unsigned', 'long', 'volatile', 'static',
+        'protected', 'bool', 'public', 'friend', 'auto', 'void', 'enum', 'extern',
+        'class', 'short', 'reinterpret_cast', 'double', 'register', 'explicit',
+        'signed', 'inline', 'delete', '_Bool', 'complex', '_Complex', '_Imaginary',
+        'atomic_bool', 'atomic_char', 'atomic_schar', 'atomic_uchar', 'atomic_short',
+        'atomic_ushort', 'atomic_int', 'atomic_uint', 'atomic_long', 'atomic_ulong',
+        'atomic_llong', 'atomic_ullong', 'PROGMEM'))
+
+    # Language shipped functions and class ( )
+    functions = set((
+        'KeyboardController', 'MouseController', 'SoftwareSerial', 'EthernetServer',
+        'EthernetClient', 'LiquidCrystal', 'RobotControl', 'GSMVoiceCall',
+        'EthernetUDP', 'EsploraTFT', 'HttpClient', 'RobotMotor', 'WiFiClient',
+        'GSMScanner', 'FileSystem', 'Scheduler', 'GSMServer', 'YunClient', 'YunServer',
+        'IPAddress', 'GSMClient', 'GSMModem', 'Keyboard', 'Ethernet', 'Console',
+        'GSMBand', 'Esplora', 'Stepper', 'Process', 'WiFiUDP', 'GSM_SMS', 'Mailbox',
+        'USBHost', 'Firmata', 'PImage', 'Client', 'Server', 'GSMPIN', 'FileIO',
+        'Bridge', 'Serial', 'EEPROM', 'Stream', 'Mouse', 'Audio', 'Servo', 'File',
+        'Task', 'GPRS', 'WiFi', 'Wire', 'TFT', 'GSM', 'SPI', 'SD',
+        'runShellCommandAsynchronously', 'analogWriteResolution',
+        'retrieveCallingNumber', 'printFirmwareVersion', 'analogReadResolution',
+        'sendDigitalPortPair', 'noListenOnLocalhost', 'readJoystickButton',
+        'setFirmwareVersion', 'readJoystickSwitch', 'scrollDisplayRight',
+        'getVoiceCallStatus', 'scrollDisplayLeft', 'writeMicroseconds',
+        'delayMicroseconds', 'beginTransmission', 'getSignalStrength',
+        'runAsynchronously', 'getAsynchronously', 'listenOnLocalhost',
+        'getCurrentCarrier', 'readAccelerometer', 'messageAvailable',
+        'sendDigitalPorts', 'lineFollowConfig', 'countryNameWrite', 'runShellCommand',
+        'readStringUntil', 'rewindDirectory', 'readTemperature', 'setClockDivider',
+        'readLightSensor', 'endTransmission', 'analogReference', 'detachInterrupt',
+        'countryNameRead', 'attachInterrupt', 'encryptionType', 'readBytesUntil',
+        'robotNameWrite', 'readMicrophone', 'robotNameRead', 'cityNameWrite',
+        'userNameWrite', 'readJoystickY', 'readJoystickX', 'mouseReleased',
+        'openNextFile', 'scanNetworks', 'noInterrupts', 'digitalWrite', 'beginSpeaker',
+        'mousePressed', 'isActionDone', 'mouseDragged', 'displayLogos', 'noAutoscroll',
+        'addParameter', 'remoteNumber', 'getModifiers', 'keyboardRead', 'userNameRead',
+        'waitContinue', 'processInput', 'parseCommand', 'printVersion', 'readNetworks',
+        'writeMessage', 'blinkVersion', 'cityNameRead', 'readMessage', 'setDataMode',
+        'parsePacket', 'isListening', 'setBitOrder', 'beginPacket', 'isDirectory',
+        'motorsWrite', 'drawCompass', 'digitalRead', 'clearScreen', 'serialEvent',
+        'rightToLeft', 'setTextSize', 'leftToRight', 'requestFrom', 'keyReleased',
+        'compassRead', 'analogWrite', 'interrupts', 'WiFiServer', 'disconnect',
+        'playMelody', 'parseFloat', 'autoscroll', 'getPINUsed', 'setPINUsed',
+        'setTimeout', 'sendAnalog', 'readSlider', 'analogRead', 'beginWrite',
+        'createChar', 'motorsStop', 'keyPressed', 'tempoWrite', 'readButton',
+        'subnetMask', 'debugPrint', 'macAddress', 'writeGreen', 'randomSeed',
+        'attachGPRS', 'readString', 'sendString', 'remotePort', 'releaseAll',
+        'mouseMoved', 'background', 'getXChange', 'getYChange', 'answerCall',
+        'getResult', 'voiceCall', 'endPacket', 'constrain', 'getSocket', 'writeJSON',
+        'getButton', 'available', 'connected', 'findUntil', 'readBytes', 'exitValue',
+        'readGreen', 'writeBlue', 'startLoop', 'IPAddress', 'isPressed', 'sendSysex',
+        'pauseMode', 'gatewayIP', 'setCursor', 'getOemKey', 'tuneWrite', 'noDisplay',
+        'loadImage', 'switchPIN', 'onRequest', 'onReceive', 'changePIN', 'playFile',
+        'noBuffer', 'parseInt', 'overflow', 'checkPIN', 'knobRead', 'beginTFT',
+        'bitClear', 'updateIR', 'bitWrite', 'position', 'writeRGB', 'highByte',
+        'writeRed', 'setSpeed', 'readBlue', 'noStroke', 'remoteIP', 'transfer',
+        'shutdown', 'hangCall', 'beginSMS', 'endWrite', 'attached', 'maintain',
+        'noCursor', 'checkReg', 'checkPUK', 'shiftOut', 'isValid', 'shiftIn', 'pulseIn',
+        'connect', 'println', 'localIP', 'pinMode', 'getIMEI', 'display', 'noBlink',
+        'process', 'getBand', 'running', 'beginSD', 'drawBMP', 'lowByte', 'setBand',
+        'release', 'bitRead', 'prepare', 'pointTo', 'readRed', 'setMode', 'noFill',
+        'remove', 'listen', 'stroke', 'detach', 'attach', 'noTone', 'exists', 'buffer',
+        'height', 'bitSet', 'circle', 'config', 'cursor', 'random', 'IRread', 'setDNS',
+        'endSMS', 'getKey', 'micros', 'millis', 'begin', 'print', 'write', 'ready',
+        'flush', 'width', 'isPIN', 'blink', 'clear', 'press', 'mkdir', 'rmdir', 'close',
+        'point', 'yield', 'image', 'BSSID', 'click', 'delay', 'read', 'text', 'move',
+        'peek', 'beep', 'rect', 'line', 'open', 'seek', 'fill', 'size', 'turn', 'stop',
+        'home', 'find', 'step', 'tone', 'sqrt', 'RSSI', 'SSID', 'end', 'bit', 'tan',
+        'cos', 'sin', 'pow', 'map', 'abs', 'max', 'min', 'get', 'run', 'put',
+        'isAlphaNumeric', 'isAlpha', 'isAscii', 'isWhitespace', 'isControl', 'isDigit', 
+        'isGraph', 'isLowerCase', 'isPrintable', 'isPunct', 'isSpace', 'isUpperCase', 
+        'isHexadecimalDigit'))    
+
+    # do not highlight
+    suppress_highlight = set((
+        'namespace', 'template', 'mutable', 'using', 'asm', 'typeid', 
+        'typename', 'this', 'alignof', 'constexpr', 'decltype', 'noexcept', 
+        'static_assert', 'thread_local', 'restrict'))
+
+
+    def get_tokens_unprocessed(self, text):
+        for index, token, value in CppLexer.get_tokens_unprocessed(self, text):
+            if value in self.structure:
+                yield index, Name.Builtin, value
+            elif value in self.operators:
+                yield index, Operator, value
+            elif value in self.variables:
+                yield index, Keyword.Reserved, value
+            elif value in self.suppress_highlight:
+                yield index, Name, value
+            elif value in self.functions:
+                yield index, Name.Function, value
+            else:
+                yield index, token, value

packages/wakatime/packages/pygments/lexers/capnproto.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.capnproto
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the Cap'n Proto schema language.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, default
+from pygments.token import Text, Comment, Keyword, Name, Literal
+
+__all__ = ['CapnProtoLexer']
+
+
+class CapnProtoLexer(RegexLexer):
+    """
+    For `Cap'n Proto <https://capnproto.org>`_ source.
+
+    .. versionadded:: 2.2
+    """
+    name = 'Cap\'n Proto'
+    filenames = ['*.capnp']
+    aliases = ['capnp']
+
+    flags = re.MULTILINE | re.UNICODE
+
+    tokens = {
+        'root': [
+            (r'#.*?$', Comment.Single),
+            (r'@[0-9a-zA-Z]*', Name.Decorator),
+            (r'=', Literal, 'expression'),
+            (r':', Name.Class, 'type'),
+            (r'\$', Name.Attribute, 'annotation'),
+            (r'(struct|enum|interface|union|import|using|const|annotation|'
+             r'extends|in|of|on|as|with|from|fixed)\b',
+             Keyword),
+            (r'[\w.]+', Name),
+            (r'[^#@=:$\w]+', Text),
+        ],
+        'type': [
+            (r'[^][=;,(){}$]+', Name.Class),
+            (r'[[(]', Name.Class, 'parentype'),
+            default('#pop'),
+        ],
+        'parentype': [
+            (r'[^][;()]+', Name.Class),
+            (r'[[(]', Name.Class, '#push'),
+            (r'[])]', Name.Class, '#pop'),
+            default('#pop'),
+        ],
+        'expression': [
+            (r'[^][;,(){}$]+', Literal),
+            (r'[[(]', Literal, 'parenexp'),
+            default('#pop'),
+        ],
+        'parenexp': [
+            (r'[^][;()]+', Literal),
+            (r'[[(]', Literal, '#push'),
+            (r'[])]', Literal, '#pop'),
+            default('#pop'),
+        ],
+        'annotation': [
+            (r'[^][;,(){}=:]+', Name.Attribute),
+            (r'[[(]', Name.Attribute, 'annexp'),
+            default('#pop'),
+        ],
+        'annexp': [
+            (r'[^][;()]+', Name.Attribute),
+            (r'[[(]', Name.Attribute, '#push'),
+            (r'[])]', Name.Attribute, '#pop'),
+            default('#pop'),
+        ],
+    }

packages/wakatime/packages/pygments/lexers/chapel.py

@@ -5,7 +5,7 @@
 
     Lexer for the Chapel language.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -42,15 +42,17 @@ class ChapelLexer(RegexLexer):
             (r'(bool|complex|imag|int|opaque|range|real|string|uint)\b',
              Keyword.Type),
             (words((
-                'align', 'atomic', 'begin', 'break', 'by', 'cobegin', 'coforall',
-                'continue', 'delete', 'dmapped', 'do', 'domain', 'else', 'enum',
-                'export', 'extern', 'for', 'forall', 'if', 'index', 'inline',
-                'iter', 'label', 'lambda', 'let', 'local', 'new', 'noinit', 'on',
-                'otherwise', 'pragma', 'reduce', 'return', 'scan', 'select',
-                'serial', 'single', 'sparse', 'subdomain', 'sync', 'then', 'use',
-                'when', 'where', 'while', 'with', 'yield', 'zip'), suffix=r'\b'),
+                'align', 'as', 'atomic', 'begin', 'break', 'by', 'cobegin',
+                'coforall', 'continue', 'delete', 'dmapped', 'do', 'domain',
+                'else', 'enum', 'except', 'export', 'extern', 'for', 'forall',
+                'if', 'index', 'inline', 'iter', 'label', 'lambda', 'let',
+                'local', 'new', 'noinit', 'on', 'only', 'otherwise', 'pragma',
+                'private', 'public', 'reduce', 'require', 'return', 'scan',
+                'select', 'serial', 'single', 'sparse', 'subdomain', 'sync',
+                'then', 'use', 'when', 'where', 'while', 'with', 'yield',
+                'zip'), suffix=r'\b'),
              Keyword),
-            (r'(proc)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'procname'),
+            (r'(proc)((?:\s)+)', bygroups(Keyword, Text), 'procname'),
             (r'(class|module|record|union)(\s+)', bygroups(Keyword, Text),
              'classname'),
 
@@ -76,7 +78,8 @@ class ChapelLexer(RegexLexer):
             (r'[0-9]+', Number.Integer),
 
             # strings
-            (r'["\'](\\\\|\\"|[^"\'])*["\']', String),
+            (r'"(\\\\|\\"|[^"])*"', String),
+            (r"'(\\\\|\\'|[^'])*'", String),
 
             # tokens
             (r'(=|\+=|-=|\*=|/=|\*\*=|%=|&=|\|=|\^=|&&=|\|\|=|<<=|>>=|'
@@ -93,6 +96,7 @@ class ChapelLexer(RegexLexer):
             (r'[a-zA-Z_][\w$]*', Name.Class, '#pop'),
         ],
         'procname': [
-            (r'[a-zA-Z_][\w$]*', Name.Function, '#pop'),
+            (r'([a-zA-Z_][\w$]+|\~[a-zA-Z_][\w$]+|[+*/!~%<>=&^|\-]{1,2})',
+             Name.Function, '#pop'),
         ],
     }

packages/wakatime/packages/pygments/lexers/clean.py

@@ -0,0 +1,288 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.clean
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for the Clean language.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import ExtendedRegexLexer, LexerContext, \
+    bygroups, words, include, default
+from pygments.token import Comment, Keyword, Literal, Name, Number, Operator, \
+    Punctuation, String, Text, Whitespace
+
+__all__ = ['CleanLexer']
+
+
+class CleanLexer(ExtendedRegexLexer):
+    """
+    Lexer for the general purpose, state-of-the-art, pure and lazy functional
+    programming language Clean (http://clean.cs.ru.nl/Clean).
+
+    .. versionadded: 2.2
+    """
+    name = 'Clean'
+    aliases = ['clean']
+    filenames = ['*.icl', '*.dcl']
+
+    def get_tokens_unprocessed(self, text=None, context=None):
+        ctx = LexerContext(text, 0)
+        ctx.indent = 0
+        return ExtendedRegexLexer.get_tokens_unprocessed(self, text, context=ctx)
+
+    def check_class_not_import(lexer, match, ctx):
+        if match.group(0) == 'import':
+            yield match.start(), Keyword.Namespace, match.group(0)
+            ctx.stack = ctx.stack[:-1] + ['fromimportfunc']
+        else:
+            yield match.start(), Name.Class, match.group(0)
+        ctx.pos = match.end()
+
+    def check_instance_class(lexer, match, ctx):
+        if match.group(0) == 'instance' or match.group(0) == 'class':
+            yield match.start(), Keyword, match.group(0)
+        else:
+            yield match.start(), Name.Function, match.group(0)
+            ctx.stack = ctx.stack + ['fromimportfunctype']
+        ctx.pos = match.end()
+
+    @staticmethod
+    def indent_len(text):
+        # Tabs are four spaces:
+        # https://svn.cs.ru.nl/repos/clean-platform/trunk/doc/STANDARDS.txt
+        text = text.replace('\n', '')
+        return len(text.replace('\t', '    ')), len(text)
+
+    def store_indent(lexer, match, ctx):
+        ctx.indent, _ = CleanLexer.indent_len(match.group(0))
+        ctx.pos = match.end()
+        yield match.start(), Text, match.group(0)
+
+    def check_indent1(lexer, match, ctx):
+        indent, reallen = CleanLexer.indent_len(match.group(0))
+        if indent > ctx.indent:
+            yield match.start(), Whitespace, match.group(0)
+            ctx.pos = match.start() + reallen + 1
+        else:
+            ctx.indent = 0
+            ctx.pos = match.start()
+            ctx.stack = ctx.stack[:-1]
+            yield match.start(), Whitespace, match.group(0)[1:]
+
+    def check_indent2(lexer, match, ctx):
+        indent, reallen = CleanLexer.indent_len(match.group(0))
+        if indent > ctx.indent:
+            yield match.start(), Whitespace, match.group(0)
+            ctx.pos = match.start() + reallen + 1
+        else:
+            ctx.indent = 0
+            ctx.pos = match.start()
+            ctx.stack = ctx.stack[:-2]
+
+    def check_indent3(lexer, match, ctx):
+        indent, reallen = CleanLexer.indent_len(match.group(0))
+        if indent > ctx.indent:
+            yield match.start(), Whitespace, match.group(0)
+            ctx.pos = match.start() + reallen + 1
+        else:
+            ctx.indent = 0
+            ctx.pos = match.start()
+            ctx.stack = ctx.stack[:-3]
+            yield match.start(), Whitespace, match.group(0)[1:]
+            if match.group(0) == '\n\n':
+                ctx.pos = ctx.pos + 1
+
+    def skip(lexer, match, ctx):
+        ctx.stack = ctx.stack[:-1]
+        ctx.pos = match.end()
+        yield match.start(), Comment, match.group(0)
+
+    keywords = ('class', 'instance', 'where', 'with', 'let', 'let!',
+                'in', 'case', 'of', 'infix', 'infixr', 'infixl', 'generic',
+                'derive', 'otherwise', 'code', 'inline')
+
+    tokens = {
+        'common': [
+            (r';', Punctuation, '#pop'),
+            (r'//', Comment, 'singlecomment'),
+        ],
+        'root': [
+            # Comments
+            (r'//.*\n', Comment.Single),
+            (r'(?s)/\*\*.*?\*/', Comment.Special),
+            (r'(?s)/\*.*?\*/', Comment.Multi),
+
+            # Modules, imports, etc.
+            (r'\b((?:implementation|definition|system)\s+)?(module)(\s+)([\w`.]+)',
+                bygroups(Keyword.Namespace, Keyword.Namespace, Text, Name.Class)),
+            (r'(?<=\n)import(?=\s)', Keyword.Namespace, 'import'),
+            (r'(?<=\n)from(?=\s)', Keyword.Namespace, 'fromimport'),
+
+            # Keywords
+            # We cannot use (?s)^|(?<=\s) as prefix, so need to repeat this
+            (words(keywords, prefix=r'(?<=\s)', suffix=r'(?=\s)'), Keyword),
+            (words(keywords, prefix=r'^', suffix=r'(?=\s)'), Keyword),
+
+            # Function definitions
+            (r'(?=\{\|)', Whitespace, 'genericfunction'),
+            (r'(?<=\n)([ \t]*)([\w`$()=\-<>~*\^|+&%]+)((?:\s+\w)*)(\s*)(::)',
+             bygroups(store_indent, Name.Function, Keyword.Type, Whitespace,
+                      Punctuation),
+             'functiondefargs'),
+
+            # Type definitions
+            (r'(?<=\n)([ \t]*)(::)', bygroups(store_indent, Punctuation), 'typedef'),
+            (r'^([ \t]*)(::)', bygroups(store_indent, Punctuation), 'typedef'),
+
+            # Literals
+            (r'\'\\?.(?<!\\)\'', String.Char),
+            (r'\'\\\d+\'', String.Char),
+            (r'\'\\\\\'', String.Char),  # (special case for '\\')
+            (r'[+\-~]?\s*\d+\.\d+(E[+\-~]?\d+)?\b', Number.Float),
+            (r'[+\-~]?\s*0[0-7]\b', Number.Oct),
+            (r'[+\-~]?\s*0x[0-9a-fA-F]\b', Number.Hex),
+            (r'[+\-~]?\s*\d+\b', Number.Integer),
+            (r'"', String.Double, 'doubleqstring'),
+            (words(('True', 'False'), prefix=r'(?<=\s)', suffix=r'(?=\s)'),
+             Literal),
+
+            # Qualified names
+            (r'(\')([\w.]+)(\'\.)',
+                bygroups(Punctuation, Name.Namespace, Punctuation)),
+
+            # Everything else is some name
+            (r'([\w`$%/?@]+\.?)*[\w`$%/?@]+', Name),
+
+            # Punctuation
+            (r'[{}()\[\],:;.#]', Punctuation),
+            (r'[+\-=!<>|&~*\^/]', Operator),
+            (r'\\\\', Operator),
+
+            # Lambda expressions
+            (r'\\.*?(->|\.|=)', Name.Function),
+
+            # Whitespace
+            (r'\s', Whitespace),
+
+            include('common'),
+        ],
+        'fromimport': [
+            include('common'),
+            (r'([\w`.]+)', check_class_not_import),
+            (r'\n', Whitespace, '#pop'),
+            (r'\s', Whitespace),
+        ],
+        'fromimportfunc': [
+            include('common'),
+            (r'(::)(\s+)([^,\s]+)', bygroups(Punctuation, Text, Keyword.Type)),
+            (r'([\w`$()=\-<>~*\^|+&%/]+)', check_instance_class),
+            (r',', Punctuation),
+            (r'\n', Whitespace, '#pop'),
+            (r'\s', Whitespace),
+        ],
+        'fromimportfunctype': [
+            include('common'),
+            (r'[{(\[]', Punctuation, 'combtype'),
+            (r',', Punctuation, '#pop'),
+            (r'[:;.#]', Punctuation),
+            (r'\n', Whitespace, '#pop:2'),
+            (r'[^\S\n]+', Whitespace),
+            (r'\S+', Keyword.Type),
+        ],
+        'combtype': [
+            include('common'),
+            (r'[})\]]', Punctuation, '#pop'),
+            (r'[{(\[]', Punctuation, '#pop'),
+            (r'[,:;.#]', Punctuation),
+            (r'\s+', Whitespace),
+            (r'\S+', Keyword.Type),
+        ],
+        'import': [
+            include('common'),
+            (words(('from', 'import', 'as', 'qualified'),
+                   prefix='(?<=\s)', suffix='(?=\s)'), Keyword.Namespace),
+            (r'[\w`.]+', Name.Class),
+            (r'\n', Whitespace, '#pop'),
+            (r',', Punctuation),
+            (r'[^\S\n]+', Whitespace),
+        ],
+        'singlecomment': [
+            (r'(.)(?=\n)', skip),
+            (r'.+(?!\n)', Comment),
+        ],
+        'doubleqstring': [
+            (r'[^\\"]+', String.Double),
+            (r'"', String.Double, '#pop'),
+            (r'\\.', String.Double),
+        ],
+        'typedef': [
+            include('common'),
+            (r'[\w`]+', Keyword.Type),
+            (r'[:=|(),\[\]{}!*]', Punctuation),
+            (r'->', Punctuation),
+            (r'\n(?=[^\s|])', Whitespace, '#pop'),
+            (r'\s', Whitespace),
+            (r'.', Keyword.Type),
+        ],
+        'genericfunction': [
+            include('common'),
+            (r'\{\|', Punctuation),
+            (r'\|\}', Punctuation, '#pop'),
+            (r',', Punctuation),
+            (r'->', Punctuation),
+            (r'(\s+of\s+)(\{)', bygroups(Keyword, Punctuation), 'genericftypes'),
+            (r'\s', Whitespace),
+            (r'[\w`\[\]{}!]+', Keyword.Type),
+            (r'[*()]', Punctuation),
+        ],
+        'genericftypes': [
+            include('common'),
+            (r'[\w`]+', Keyword.Type),
+            (r',', Punctuation),
+            (r'\s', Whitespace),
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'functiondefargs': [
+            include('common'),
+            (r'\n(\s*)', check_indent1),
+            (r'[!{}()\[\],:;.#]', Punctuation),
+            (r'->', Punctuation, 'functiondefres'),
+            (r'^(?=\S)', Whitespace, '#pop'),
+            (r'\S', Keyword.Type),
+            (r'\s', Whitespace),
+        ],
+        'functiondefres': [
+            include('common'),
+            (r'\n(\s*)', check_indent2),
+            (r'^(?=\S)', Whitespace, '#pop:2'),
+            (r'[!{}()\[\],:;.#]', Punctuation),
+            (r'\|', Punctuation, 'functiondefclasses'),
+            (r'\S', Keyword.Type),
+            (r'\s', Whitespace),
+        ],
+        'functiondefclasses': [
+            include('common'),
+            (r'\n(\s*)', check_indent3),
+            (r'^(?=\S)', Whitespace, '#pop:3'),
+            (r'[,&]', Punctuation),
+            (r'\[', Punctuation, 'functiondefuniquneq'),
+            (r'[\w`$()=\-<>~*\^|+&%/{}\[\]@]', Name.Function, 'functionname'),
+            (r'\s+', Whitespace),
+        ],
+        'functiondefuniquneq': [
+            include('common'),
+            (r'[a-z]+', Keyword.Type),
+            (r'\s+', Whitespace),
+            (r'<=|,', Punctuation),
+            (r'\]', Punctuation, '#pop')
+        ],
+        'functionname': [
+            include('common'),
+            (r'[\w`$()=\-<>~*\^|+&%/]+', Name.Function),
+            (r'(?=\{\|)', Punctuation, 'genericfunction'),
+            default('#pop'),
+        ]
+    }

packages/wakatime/packages/pygments/lexers/compiled.py

@@ -5,7 +5,7 @@
 
     Just export lexer classes previously contained in this module.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -29,5 +29,6 @@ from pygments.lexers.dylan import DylanLexer, DylanLidLexer, DylanConsoleLexer
 from pygments.lexers.ooc import OocLexer
 from pygments.lexers.felix import FelixLexer
 from pygments.lexers.nimrod import NimrodLexer
+from pygments.lexers.crystal import CrystalLexer
 
 __all__ = []

packages/wakatime/packages/pygments/lexers/configs.py

@@ -5,7 +5,7 @@
 
     Lexers for configuration file formats.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -13,12 +13,14 @@ import re
 
 from pygments.lexer import RegexLexer, default, words, bygroups, include, using
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
-    Number, Punctuation, Whitespace
+    Number, Punctuation, Whitespace, Literal
 from pygments.lexers.shell import BashLexer
 
 __all__ = ['IniLexer', 'RegeditLexer', 'PropertiesLexer', 'KconfigLexer',
            'Cfengine3Lexer', 'ApacheConfLexer', 'SquidConfLexer',
-           'NginxConfLexer', 'LighttpdConfLexer', 'DockerLexer']
+           'NginxConfLexer', 'LighttpdConfLexer', 'DockerLexer',
+           'TerraformLexer', 'TermcapLexer', 'TerminfoLexer',
+           'PkgConfigLexer', 'PacmanConfLexer']
 
 
 class IniLexer(RegexLexer):
@@ -28,8 +30,8 @@ class IniLexer(RegexLexer):
 
     name = 'INI'
     aliases = ['ini', 'cfg', 'dosini']
-    filenames = ['*.ini', '*.cfg']
-    mimetypes = ['text/x-ini']
+    filenames = ['*.ini', '*.cfg', '*.inf']
+    mimetypes = ['text/x-ini', 'text/inf']
 
     tokens = {
         'root': [
@@ -37,8 +39,10 @@ class IniLexer(RegexLexer):
             (r'[;#].*', Comment.Single),
             (r'\[.*?\]$', Keyword),
             (r'(.*?)([ \t]*)(=)([ \t]*)(.*(?:\n[ \t].+)*)',
-             bygroups(Name.Attribute, Text, Operator, Text, String))
-        ]
+             bygroups(Name.Attribute, Text, Operator, Text, String)),
+            # standalone option, supported by some INI parsers
+            (r'(.+?)$', Name.Attribute),
+        ],
     }
 
     def analyse_text(text):
@@ -96,6 +100,8 @@ class PropertiesLexer(RegexLexer):
     """
     Lexer for configuration files in Java's properties format.
 
+    Note: trailing whitespace counts as part of the value as per spec
+
     .. versionadded:: 1.4
     """
 
@@ -106,10 +112,14 @@ class PropertiesLexer(RegexLexer):
 
     tokens = {
         'root': [
-            (r'\s+', Text),
-            (r'(?:[;#]|//).*$', Comment),
+            (r'^(\w+)([ \t])(\w+\s*)$', bygroups(Name.Attribute, Text, String)),
+            (r'^\w+(\\[ \t]\w*)*$', Name.Attribute),
+            (r'(^ *)([#!].*)', bygroups(Text, Comment)),
+            # More controversial comments
+            (r'(^ *)((?:;|//).*)', bygroups(Text, Comment)),
             (r'(.*?)([ \t]*)([=:])([ \t]*)(.*(?:(?<=\\)\n.*)*)',
              bygroups(Name.Attribute, Text, Operator, Text, String)),
+            (r'\s', Text),
         ],
     }
 
@@ -452,7 +462,7 @@ class NginxConfLexer(RegexLexer):
     """
     name = 'Nginx configuration file'
     aliases = ['nginx']
-    filenames = []
+    filenames = ['nginx.conf']
     mimetypes = ['text/x-nginx-conf']
 
     tokens = {
@@ -540,7 +550,284 @@ class DockerLexer(RegexLexer):
              bygroups(Name.Keyword, Whitespace, Keyword)),
             (r'^(%s)\b(.*)' % (_keywords,), bygroups(Keyword, String)),
             (r'#.*', Comment),
-            (r'RUN', Keyword), # Rest of line falls through
+            (r'RUN', Keyword),  # Rest of line falls through
             (r'(.*\\\n)*.+', using(BashLexer)),
         ],
     }
+
+
+class TerraformLexer(RegexLexer):
+    """
+    Lexer for `terraformi .tf files <https://www.terraform.io/>`_.
+
+    .. versionadded:: 2.1
+    """
+
+    name = 'Terraform'
+    aliases = ['terraform', 'tf']
+    filenames = ['*.tf']
+    mimetypes = ['application/x-tf', 'application/x-terraform']
+
+    tokens = {
+        'root': [
+             include('string'),
+             include('punctuation'),
+             include('curly'),
+             include('basic'),
+             include('whitespace'),
+             (r'[0-9]+', Number),
+        ],
+        'basic': [
+             (words(('true', 'false'), prefix=r'\b', suffix=r'\b'), Keyword.Type),
+             (r'\s*/\*', Comment.Multiline, 'comment'),
+             (r'\s*#.*\n', Comment.Single),
+             (r'(.*?)(\s*)(=)', bygroups(Name.Attribute, Text, Operator)),
+             (words(('variable', 'resource', 'provider', 'provisioner', 'module'),
+                    prefix=r'\b', suffix=r'\b'), Keyword.Reserved, 'function'),
+             (words(('ingress', 'egress', 'listener', 'default', 'connection'),
+                    prefix=r'\b', suffix=r'\b'), Keyword.Declaration),
+             ('\$\{', String.Interpol, 'var_builtin'),
+        ],
+        'function': [
+             (r'(\s+)(".*")(\s+)', bygroups(Text, String, Text)),
+             include('punctuation'),
+             include('curly'),
+        ],
+        'var_builtin': [
+            (r'\$\{', String.Interpol, '#push'),
+            (words(('concat', 'file', 'join', 'lookup', 'element'),
+                   prefix=r'\b', suffix=r'\b'), Name.Builtin),
+            include('string'),
+            include('punctuation'),
+            (r'\s+', Text),
+            (r'\}', String.Interpol, '#pop'),
+        ],
+        'string': [
+            (r'(".*")', bygroups(String.Double)),
+        ],
+        'punctuation': [
+            (r'[\[\](),.]', Punctuation),
+        ],
+        # Keep this seperate from punctuation - we sometimes want to use different
+        # Tokens for { }
+        'curly': [
+            (r'\{', Text.Punctuation),
+            (r'\}', Text.Punctuation),
+        ],
+        'comment': [
+            (r'[^*/]', Comment.Multiline),
+            (r'/\*', Comment.Multiline, '#push'),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'[*/]', Comment.Multiline)
+        ],
+        'whitespace': [
+            (r'\n', Text),
+            (r'\s+', Text),
+            (r'\\\n', Text),
+        ],
+    }
+
+
+class TermcapLexer(RegexLexer):
+    """
+    Lexer for termcap database source.
+
+    This is very simple and minimal.
+
+    .. versionadded:: 2.1
+    """
+    name = 'Termcap'
+    aliases = ['termcap']
+    filenames = ['termcap', 'termcap.src']
+    mimetypes = []
+
+    # NOTE:
+    #   * multiline with trailing backslash
+    #   * separator is ':'
+    #   * to embed colon as data, we must use \072
+    #   * space after separator is not allowed (mayve)
+    tokens = {
+        'root': [
+            (r'^#.*$', Comment),
+            (r'^[^\s#:|]+', Name.Tag, 'names'),
+        ],
+        'names': [
+            (r'\n', Text, '#pop'),
+            (r':', Punctuation, 'defs'),
+            (r'\|', Punctuation),
+            (r'[^:|]+', Name.Attribute),
+        ],
+        'defs': [
+            (r'\\\n[ \t]*', Text),
+            (r'\n[ \t]*', Text, '#pop:2'),
+            (r'(#)([0-9]+)', bygroups(Operator, Number)),
+            (r'=', Operator, 'data'),
+            (r':', Punctuation),
+            (r'[^\s:=#]+', Name.Class),
+        ],
+        'data': [
+            (r'\\072', Literal),
+            (r':', Punctuation, '#pop'),
+            (r'[^:\\]+', Literal),  # for performance
+            (r'.', Literal),
+        ],
+    }
+
+
+class TerminfoLexer(RegexLexer):
+    """
+    Lexer for terminfo database source.
+
+    This is very simple and minimal.
+
+    .. versionadded:: 2.1
+    """
+    name = 'Terminfo'
+    aliases = ['terminfo']
+    filenames = ['terminfo', 'terminfo.src']
+    mimetypes = []
+
+    # NOTE:
+    #   * multiline with leading whitespace
+    #   * separator is ','
+    #   * to embed comma as data, we can use \,
+    #   * space after separator is allowed
+    tokens = {
+        'root': [
+            (r'^#.*$', Comment),
+            (r'^[^\s#,|]+', Name.Tag, 'names'),
+        ],
+        'names': [
+            (r'\n', Text, '#pop'),
+            (r'(,)([ \t]*)', bygroups(Punctuation, Text), 'defs'),
+            (r'\|', Punctuation),
+            (r'[^,|]+', Name.Attribute),
+        ],
+        'defs': [
+            (r'\n[ \t]+', Text),
+            (r'\n', Text, '#pop:2'),
+            (r'(#)([0-9]+)', bygroups(Operator, Number)),
+            (r'=', Operator, 'data'),
+            (r'(,)([ \t]*)', bygroups(Punctuation, Text)),
+            (r'[^\s,=#]+', Name.Class),
+        ],
+        'data': [
+            (r'\\[,\\]', Literal),
+            (r'(,)([ \t]*)', bygroups(Punctuation, Text), '#pop'),
+            (r'[^\\,]+', Literal),  # for performance
+            (r'.', Literal),
+        ],
+    }
+
+
+class PkgConfigLexer(RegexLexer):
+    """
+    Lexer for `pkg-config
+    <http://www.freedesktop.org/wiki/Software/pkg-config/>`_
+    (see also `manual page <http://linux.die.net/man/1/pkg-config>`_).
+
+    .. versionadded:: 2.1
+    """
+
+    name = 'PkgConfig'
+    aliases = ['pkgconfig']
+    filenames = ['*.pc']
+    mimetypes = []
+
+    tokens = {
+        'root': [
+            (r'#.*$', Comment.Single),
+
+            # variable definitions
+            (r'^(\w+)(=)', bygroups(Name.Attribute, Operator)),
+
+            # keyword lines
+            (r'^([\w.]+)(:)',
+             bygroups(Name.Tag, Punctuation), 'spvalue'),
+
+            # variable references
+            include('interp'),
+
+            # fallback
+            (r'[^${}#=:\n.]+', Text),
+            (r'.', Text),
+        ],
+        'interp': [
+            # you can escape literal "$" as "$$"
+            (r'\$\$', Text),
+
+            # variable references
+            (r'\$\{', String.Interpol, 'curly'),
+        ],
+        'curly': [
+            (r'\}', String.Interpol, '#pop'),
+            (r'\w+', Name.Attribute),
+        ],
+        'spvalue': [
+            include('interp'),
+
+            (r'#.*$', Comment.Single, '#pop'),
+            (r'\n', Text, '#pop'),
+
+            # fallback
+            (r'[^${}#\n]+', Text),
+            (r'.', Text),
+        ],
+    }
+
+
+class PacmanConfLexer(RegexLexer):
+    """
+    Lexer for `pacman.conf
+    <https://www.archlinux.org/pacman/pacman.conf.5.html>`_.
+
+    Actually, IniLexer works almost fine for this format,
+    but it yield error token. It is because pacman.conf has
+    a form without assignment like:
+
+        UseSyslog
+        Color
+        TotalDownload
+        CheckSpace
+        VerbosePkgLists
+
+    These are flags to switch on.
+
+    .. versionadded:: 2.1
+    """
+
+    name = 'PacmanConf'
+    aliases = ['pacmanconf']
+    filenames = ['pacman.conf']
+    mimetypes = []
+
+    tokens = {
+        'root': [
+            # comment
+            (r'#.*$', Comment.Single),
+
+            # section header
+            (r'^\s*\[.*?\]\s*$', Keyword),
+
+            # variable definitions
+            # (Leading space is allowed...)
+            (r'(\w+)(\s*)(=)',
+             bygroups(Name.Attribute, Text, Operator)),
+
+            # flags to on
+            (r'^(\s*)(\w+)(\s*)$',
+             bygroups(Text, Name.Attribute, Text)),
+
+            # built-in special values
+            (words((
+                '$repo',  # repository
+                '$arch',  # architecture
+                '%o',     # outfile
+                '%u',     # url
+                ), suffix=r'\b'),
+             Name.Variable),
+
+            # fallback
+            (r'.', Text),
+        ],
+    }

packages/wakatime/packages/pygments/lexers/console.py

@@ -5,7 +5,7 @@
 
     Lexers for misc console output.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -18,7 +18,7 @@ __all__ = ['VCTreeStatusLexer', 'PyPyLogLexer']
 
 class VCTreeStatusLexer(RegexLexer):
     """
-    For colorizing output of version control status commans, like "hg
+    For colorizing output of version control status commands, like "hg
     status" or "svn status".
 
     .. versionadded:: 2.0

packages/wakatime/packages/pygments/lexers/crystal.py

@@ -0,0 +1,393 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.crystal
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Crystal.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import ExtendedRegexLexer, include, \
+    bygroups, default, LexerContext, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Error
+
+__all__ = ['CrystalLexer']
+
+line_re = re.compile('.*?\n')
+
+
+CRYSTAL_OPERATORS = [
+    '!=', '!~', '!', '%', '&&', '&', '**', '*', '+', '-', '/', '<=>', '<<', '<=', '<',
+    '===', '==', '=~', '=', '>=', '>>', '>', '[]=', '[]?', '[]', '^', '||', '|', '~'
+]
+
+
+class CrystalLexer(ExtendedRegexLexer):
+    """
+    For `Crystal <http://crystal-lang.org>`_ source code.
+
+    .. versionadded:: 2.2
+    """
+
+    name = 'Crystal'
+    aliases = ['cr', 'crystal']
+    filenames = ['*.cr']
+    mimetypes = ['text/x-crystal']
+
+    flags = re.DOTALL | re.MULTILINE
+
+    def heredoc_callback(self, match, ctx):
+        # okay, this is the hardest part of parsing Crystal...
+        # match: 1 = <<-?, 2 = quote? 3 = name 4 = quote? 5 = rest of line
+
+        start = match.start(1)
+        yield start, Operator, match.group(1)        # <<-?
+        yield match.start(2), String.Heredoc, match.group(2)    # quote ", ', `
+        yield match.start(3), String.Delimiter, match.group(3)  # heredoc name
+        yield match.start(4), String.Heredoc, match.group(4)    # quote again
+
+        heredocstack = ctx.__dict__.setdefault('heredocstack', [])
+        outermost = not bool(heredocstack)
+        heredocstack.append((match.group(1) == '<<-', match.group(3)))
+
+        ctx.pos = match.start(5)
+        ctx.end = match.end(5)
+        # this may find other heredocs
+        for i, t, v in self.get_tokens_unprocessed(context=ctx):
+            yield i, t, v
+        ctx.pos = match.end()
+
+        if outermost:
+            # this is the outer heredoc again, now we can process them all
+            for tolerant, hdname in heredocstack:
+                lines = []
+                for match in line_re.finditer(ctx.text, ctx.pos):
+                    if tolerant:
+                        check = match.group().strip()
+                    else:
+                        check = match.group().rstrip()
+                    if check == hdname:
+                        for amatch in lines:
+                            yield amatch.start(), String.Heredoc, amatch.group()
+                        yield match.start(), String.Delimiter, match.group()
+                        ctx.pos = match.end()
+                        break
+                    else:
+                        lines.append(match)
+                else:
+                    # end of heredoc not found -- error!
+                    for amatch in lines:
+                        yield amatch.start(), Error, amatch.group()
+            ctx.end = len(ctx.text)
+            del heredocstack[:]
+
+    def gen_crystalstrings_rules():
+        def intp_regex_callback(self, match, ctx):
+            yield match.start(1), String.Regex, match.group(1)  # begin
+            nctx = LexerContext(match.group(3), 0, ['interpolated-regex'])
+            for i, t, v in self.get_tokens_unprocessed(context=nctx):
+                yield match.start(3)+i, t, v
+            yield match.start(4), String.Regex, match.group(4)  # end[imsx]*
+            ctx.pos = match.end()
+
+        def intp_string_callback(self, match, ctx):
+            yield match.start(1), String.Other, match.group(1)
+            nctx = LexerContext(match.group(3), 0, ['interpolated-string'])
+            for i, t, v in self.get_tokens_unprocessed(context=nctx):
+                yield match.start(3)+i, t, v
+            yield match.start(4), String.Other, match.group(4)  # end
+            ctx.pos = match.end()
+
+        states = {}
+        states['strings'] = [
+            (r'\:@{0,2}[a-zA-Z_]\w*[!?]?', String.Symbol),
+            (words(CRYSTAL_OPERATORS, prefix=r'\:@{0,2}'), String.Symbol),
+            (r":'(\\\\|\\'|[^'])*'", String.Symbol),
+            # This allows arbitrary text after '\ for simplicity
+            (r"'(\\\\|\\'|[^']|\\[^'\\]+)'", String.Char),
+            (r':"', String.Symbol, 'simple-sym'),
+            # Crystal doesn't have "symbol:"s but this simplifies function args
+            (r'([a-zA-Z_]\w*)(:)(?!:)', bygroups(String.Symbol, Punctuation)),
+            (r'"', String.Double, 'simple-string'),
+            (r'(?<!\.)`', String.Backtick, 'simple-backtick'),
+        ]
+
+        # double-quoted string and symbol
+        for name, ttype, end in ('string', String.Double, '"'), \
+                                ('sym', String.Symbol, '"'), \
+                                ('backtick', String.Backtick, '`'):
+            states['simple-'+name] = [
+                include('string-escaped' if name == 'sym' else 'string-intp-escaped'),
+                (r'[^\\%s#]+' % end, ttype),
+                (r'[\\#]', ttype),
+                (end, ttype, '#pop'),
+            ]
+
+        # braced quoted strings
+        for lbrace, rbrace, bracecc, name in \
+                ('\\{', '\\}', '{}', 'cb'), \
+                ('\\[', '\\]', '\\[\\]', 'sb'), \
+                ('\\(', '\\)', '()', 'pa'), \
+                ('<', '>', '<>', 'ab'):
+            states[name+'-intp-string'] = [
+                (r'\\[' + lbrace + ']', String.Other),
+                (lbrace, String.Other, '#push'),
+                (rbrace, String.Other, '#pop'),
+                include('string-intp-escaped'),
+                (r'[\\#' + bracecc + ']', String.Other),
+                (r'[^\\#' + bracecc + ']+', String.Other),
+            ]
+            states['strings'].append((r'%' + lbrace, String.Other,
+                                      name+'-intp-string'))
+            states[name+'-string'] = [
+                (r'\\[\\' + bracecc + ']', String.Other),
+                (lbrace, String.Other, '#push'),
+                (rbrace, String.Other, '#pop'),
+                (r'[\\#' + bracecc + ']', String.Other),
+                (r'[^\\#' + bracecc + ']+', String.Other),
+            ]
+            # http://crystal-lang.org/docs/syntax_and_semantics/literals/array.html
+            states['strings'].append((r'%[wi]' + lbrace, String.Other,
+                                      name+'-string'))
+            states[name+'-regex'] = [
+                (r'\\[\\' + bracecc + ']', String.Regex),
+                (lbrace, String.Regex, '#push'),
+                (rbrace + '[imsx]*', String.Regex, '#pop'),
+                include('string-intp'),
+                (r'[\\#' + bracecc + ']', String.Regex),
+                (r'[^\\#' + bracecc + ']+', String.Regex),
+            ]
+            states['strings'].append((r'%r' + lbrace, String.Regex,
+                                      name+'-regex'))
+
+        # these must come after %<brace>!
+        states['strings'] += [
+            # %r regex
+            (r'(%r([\W_]))((?:\\\2|(?!\2).)*)(\2[imsx]*)',
+             intp_regex_callback),
+            # regular fancy strings with qsw
+            (r'(%[wi]([\W_]))((?:\\\2|(?!\2).)*)(\2)',
+             intp_string_callback),
+            # special forms of fancy strings after operators or
+            # in method calls with braces
+            (r'(?<=[-+/*%=<>&!^|~,(])(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
+             bygroups(Text, String.Other, None)),
+            # and because of fixed width lookbehinds the whole thing a
+            # second time for line startings...
+            (r'^(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
+             bygroups(Text, String.Other, None)),
+            # all regular fancy strings without qsw
+            (r'(%([\[{(<]))((?:\\\2|(?!\2).)*)(\2)',
+             intp_string_callback),
+        ]
+
+        return states
+
+    tokens = {
+        'root': [
+            (r'#.*?$', Comment.Single),
+            # keywords
+            (words('''
+                abstract asm as begin break case do else elsif end ensure extend ifdef if
+                include instance_sizeof next of pointerof private protected rescue return
+                require sizeof super then typeof unless until when while with yield
+            '''.split(), suffix=r'\b'), Keyword),
+            (words(['true', 'false', 'nil'], suffix=r'\b'), Keyword.Constant),
+            # start of function, class and module names
+            (r'(module|lib)(\s+)([a-zA-Z_]\w*(?:::[a-zA-Z_]\w*)*)',
+             bygroups(Keyword, Text, Name.Namespace)),
+            (r'(def|fun|macro)(\s+)((?:[a-zA-Z_]\w*::)*)',
+             bygroups(Keyword, Text, Name.Namespace), 'funcname'),
+            (r'def(?=[*%&^`~+-/\[<>=])', Keyword, 'funcname'),
+            (r'(class|struct|union|type|alias|enum)(\s+)((?:[a-zA-Z_]\w*::)*)',
+             bygroups(Keyword, Text, Name.Namespace), 'classname'),
+            (r'(self|out|uninitialized)\b|(is_a|responds_to)\?', Keyword.Pseudo),
+            # macros
+            (words('''
+                debugger record pp assert_responds_to spawn parallel
+                getter setter property delegate def_hash def_equals def_equals_and_hash
+                forward_missing_to
+            '''.split(), suffix=r'\b'), Name.Builtin.Pseudo),
+            (r'getter[!?]|property[!?]|__(DIR|FILE|LINE)__\b', Name.Builtin.Pseudo),
+            # builtins
+            # http://crystal-lang.org/api/toplevel.html
+            (words('''
+                Object Value Struct Reference Proc Class Nil Symbol Enum Void
+                Bool Number Int Int8 Int16 Int32 Int64 UInt8 UInt16 UInt32 UInt64
+                Float Float32 Float64 Char String
+                Pointer Slice Range Exception Regex
+                Mutex StaticArray Array Hash Set Tuple Deque Box Process File
+                Dir Time Channel Concurrent Scheduler
+                abort at_exit caller delay exit fork future get_stack_top gets
+                lazy loop main p print printf puts
+                raise rand read_line sleep sprintf system with_color
+            '''.split(), prefix=r'(?<!\.)', suffix=r'\b'), Name.Builtin),
+            # normal heredocs
+            (r'(?<!\w)(<<-?)(["`\']?)([a-zA-Z_]\w*)(\2)(.*?\n)',
+             heredoc_callback),
+            # empty string heredocs
+            (r'(<<-?)("|\')()(\2)(.*?\n)', heredoc_callback),
+            (r'__END__', Comment.Preproc, 'end-part'),
+            # multiline regex (after keywords or assignments)
+            (r'(?:^|(?<=[=<>~!:])|'
+             r'(?<=(?:\s|;)when\s)|'
+             r'(?<=(?:\s|;)or\s)|'
+             r'(?<=(?:\s|;)and\s)|'
+             r'(?<=\.index\s)|'
+             r'(?<=\.scan\s)|'
+             r'(?<=\.sub\s)|'
+             r'(?<=\.sub!\s)|'
+             r'(?<=\.gsub\s)|'
+             r'(?<=\.gsub!\s)|'
+             r'(?<=\.match\s)|'
+             r'(?<=(?:\s|;)if\s)|'
+             r'(?<=(?:\s|;)elsif\s)|'
+             r'(?<=^when\s)|'
+             r'(?<=^index\s)|'
+             r'(?<=^scan\s)|'
+             r'(?<=^sub\s)|'
+             r'(?<=^gsub\s)|'
+             r'(?<=^sub!\s)|'
+             r'(?<=^gsub!\s)|'
+             r'(?<=^match\s)|'
+             r'(?<=^if\s)|'
+             r'(?<=^elsif\s)'
+             r')(\s*)(/)', bygroups(Text, String.Regex), 'multiline-regex'),
+            # multiline regex (in method calls or subscripts)
+            (r'(?<=\(|,|\[)/', String.Regex, 'multiline-regex'),
+            # multiline regex (this time the funny no whitespace rule)
+            (r'(\s+)(/)(?![\s=])', bygroups(Text, String.Regex),
+             'multiline-regex'),
+            # lex numbers and ignore following regular expressions which
+            # are division operators in fact (grrrr. i hate that. any
+            # better ideas?)
+            # since pygments 0.7 we also eat a "?" operator after numbers
+            # so that the char operator does not work. Chars are not allowed
+            # there so that you can use the ternary operator.
+            # stupid example:
+            #   x>=0?n[x]:""
+            (r'(0o[0-7]+(?:_[0-7]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?',
+             bygroups(Number.Oct, Text, Operator)),
+            (r'(0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?',
+             bygroups(Number.Hex, Text, Operator)),
+            (r'(0b[01]+(?:_[01]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?',
+             bygroups(Number.Bin, Text, Operator)),
+            # 3 separate expressions for floats because any of the 3 optional
+            # parts makes it a float
+            (r'((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)(?:e[+-]?[0-9]+)?'
+             r'(?:_?f[0-9]+)?)(\s*)([/?])?',
+             bygroups(Number.Float, Text, Operator)),
+            (r'((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)?(?:e[+-]?[0-9]+)'
+             r'(?:_?f[0-9]+)?)(\s*)([/?])?',
+             bygroups(Number.Float, Text, Operator)),
+            (r'((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)?(?:e[+-]?[0-9]+)?'
+             r'(?:_?f[0-9]+))(\s*)([/?])?',
+             bygroups(Number.Float, Text, Operator)),
+            (r'(0\b|[1-9][\d]*(?:_\d+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?',
+             bygroups(Number.Integer, Text, Operator)),
+            # Names
+            (r'@@[a-zA-Z_]\w*', Name.Variable.Class),
+            (r'@[a-zA-Z_]\w*', Name.Variable.Instance),
+            (r'\$\w+', Name.Variable.Global),
+            (r'\$[!@&`\'+~=/\\,;.<>_*$?:"^-]', Name.Variable.Global),
+            (r'\$-[0adFiIlpvw]', Name.Variable.Global),
+            (r'::', Operator),
+            include('strings'),
+            # chars
+            (r'\?(\\[MC]-)*'  # modifiers
+             r'(\\([\\befnrtv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})|\S)'
+             r'(?!\w)',
+             String.Char),
+            (r'[A-Z][A-Z_]+\b', Name.Constant),
+            # macro expansion
+            (r'\{%', String.Interpol, 'in-macro-control'),
+            (r'\{\{', String.Interpol, 'in-macro-expr'),
+            # attributes
+            (r'(@\[)(\s*)([A-Z]\w*)',
+             bygroups(Operator, Text, Name.Decorator), 'in-attr'),
+            # this is needed because Crystal attributes can look
+            # like keywords (class) or like this: ` ?!?
+            (words(CRYSTAL_OPERATORS, prefix=r'(\.|::)'),
+             bygroups(Operator, Name.Operator)),
+            (r'(\.|::)([a-zA-Z_]\w*[!?]?|[*%&^`~+\-/\[<>=])',
+             bygroups(Operator, Name)),
+            # Names can end with [!?] unless it's "!="
+            (r'[a-zA-Z_]\w*(?:[!?](?!=))?', Name),
+            (r'(\[|\]\??|\*\*|<=>?|>=|<<?|>>?|=~|===|'
+             r'!~|&&?|\|\||\.{1,3})', Operator),
+            (r'[-+/*%=<>&!^|~]=?', Operator),
+            (r'[(){};,/?:\\]', Punctuation),
+            (r'\s+', Text)
+        ],
+        'funcname': [
+            (r'(?:([a-zA-Z_]\w*)(\.))?'
+             r'([a-zA-Z_]\w*[!?]?|\*\*?|[-+]@?|'
+             r'[/%&|^`~]|\[\]=?|<<|>>|<=?>|>=?|===?)',
+             bygroups(Name.Class, Operator, Name.Function), '#pop'),
+            default('#pop')
+        ],
+        'classname': [
+            (r'[A-Z_]\w*', Name.Class),
+            (r'(\()(\s*)([A-Z_]\w*)(\s*)(\))',
+             bygroups(Punctuation, Text, Name.Class, Text, Punctuation)),
+            default('#pop')
+        ],
+        'in-intp': [
+            (r'\{', String.Interpol, '#push'),
+            (r'\}', String.Interpol, '#pop'),
+            include('root'),
+        ],
+        'string-intp': [
+            (r'#\{', String.Interpol, 'in-intp'),
+        ],
+        'string-escaped': [
+            (r'\\([\\befnstv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})', String.Escape)
+        ],
+        'string-intp-escaped': [
+            include('string-intp'),
+            include('string-escaped'),
+        ],
+        'interpolated-regex': [
+            include('string-intp'),
+            (r'[\\#]', String.Regex),
+            (r'[^\\#]+', String.Regex),
+        ],
+        'interpolated-string': [
+            include('string-intp'),
+            (r'[\\#]', String.Other),
+            (r'[^\\#]+', String.Other),
+        ],
+        'multiline-regex': [
+            include('string-intp'),
+            (r'\\\\', String.Regex),
+            (r'\\/', String.Regex),
+            (r'[\\#]', String.Regex),
+            (r'[^\\/#]+', String.Regex),
+            (r'/[imsx]*', String.Regex, '#pop'),
+        ],
+        'end-part': [
+            (r'.+', Comment.Preproc, '#pop')
+        ],
+        'in-macro-control': [
+            (r'\{%', String.Interpol, '#push'),
+            (r'%\}', String.Interpol, '#pop'),
+            (r'for\b|in\b', Keyword),
+            include('root'),
+        ],
+        'in-macro-expr': [
+            (r'\{\{', String.Interpol, '#push'),
+            (r'\}\}', String.Interpol, '#pop'),
+            include('root'),
+        ],
+        'in-attr': [
+            (r'\[', Operator, '#push'),
+            (r'\]', Operator, '#pop'),
+            include('root'),
+        ],
+    }
+    tokens.update(gen_crystalstrings_rules())

packages/wakatime/packages/pygments/lexers/csound.py

@@ -0,0 +1,366 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.csound
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for CSound languages.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, default, include, using, words
+from pygments.token import Comment, Keyword, Name, Number, Operator, Punctuation, \
+    String, Text
+from pygments.lexers._csound_builtins import OPCODES
+from pygments.lexers.html import HtmlLexer
+from pygments.lexers.python import PythonLexer
+from pygments.lexers.scripting import LuaLexer
+
+__all__ = ['CsoundScoreLexer', 'CsoundOrchestraLexer', 'CsoundDocumentLexer']
+
+newline = (r'((?:(?:;|//).*)*)(\n)', bygroups(Comment.Single, Text))
+
+
+class CsoundLexer(RegexLexer):
+    # Subclasses must define a 'single-line string' state.
+    tokens = {
+        'whitespace': [
+            (r'[ \t]+', Text),
+            (r'\\\n', Text),
+            (r'/[*](.|\n)*?[*]/', Comment.Multiline)
+        ],
+
+        'macro call': [
+            (r'(\$\w+\.?)(\()', bygroups(Comment.Preproc, Punctuation),
+             'function macro call'),
+            (r'\$\w+(\.|\b)', Comment.Preproc)
+        ],
+        'function macro call': [
+            (r"((?:\\['\)]|[^'\)])+)(')", bygroups(Comment.Preproc, Punctuation)),
+            (r"([^'\)]+)(\))", bygroups(Comment.Preproc, Punctuation), '#pop')
+        ],
+
+        'whitespace or macro call': [
+            include('whitespace'),
+            include('macro call')
+        ],
+
+        'preprocessor directives': [
+            (r'#(e(nd(if)?|lse)|ifn?def|undef)\b|##', Comment.Preproc),
+            (r'#include\b', Comment.Preproc, 'include'),
+            (r'#[ \t]*define\b', Comment.Preproc, 'macro name'),
+            (r'@+[ \t]*\d*', Comment.Preproc)
+        ],
+
+        'include': [
+            include('whitespace'),
+            (r'"', String, 'single-line string')
+        ],
+
+        'macro name': [
+            include('whitespace'),
+            (r'(\w+)(\()', bygroups(Comment.Preproc, Text),
+             'function macro argument list'),
+            (r'\w+', Comment.Preproc, 'object macro definition after name')
+        ],
+        'object macro definition after name': [
+            include('whitespace'),
+            (r'#', Punctuation, 'object macro replacement text')
+        ],
+        'object macro replacement text': [
+            (r'(\\#|[^#])+', Comment.Preproc),
+            (r'#', Punctuation, '#pop:3')
+        ],
+        'function macro argument list': [
+            (r"(\w+)(['#])", bygroups(Comment.Preproc, Punctuation)),
+            (r'(\w+)(\))', bygroups(Comment.Preproc, Punctuation),
+             'function macro definition after name')
+        ],
+        'function macro definition after name': [
+            (r'[ \t]+', Text),
+            (r'#', Punctuation, 'function macro replacement text')
+        ],
+        'function macro replacement text': [
+            (r'(\\#|[^#])+', Comment.Preproc),
+            (r'#', Punctuation, '#pop:4')
+        ]
+    }
+
+
+class CsoundScoreLexer(CsoundLexer):
+    """
+    For `Csound <http://csound.github.io>`_ scores.
+
+    .. versionadded:: 2.1
+    """
+
+    name = 'Csound Score'
+    aliases = ['csound-score', 'csound-sco']
+    filenames = ['*.sco']
+
+    tokens = {
+        'partial statement': [
+            include('preprocessor directives'),
+            (r'\d+e[+-]?\d+|(\d+\.\d*|\d*\.\d+)(e[+-]?\d+)?', Number.Float),
+            (r'0[xX][a-fA-F0-9]+', Number.Hex),
+            (r'\d+', Number.Integer),
+            (r'"', String, 'single-line string'),
+            (r'[+\-*/%^!=<>|&#~.]', Operator),
+            (r'[]()[]', Punctuation),
+            (r'\w+', Comment.Preproc)
+        ],
+
+        'statement': [
+            include('whitespace or macro call'),
+            newline + ('#pop',),
+            include('partial statement')
+        ],
+
+        'root': [
+            newline,
+            include('whitespace or macro call'),
+            (r'[{}]', Punctuation, 'statement'),
+            (r'[abefimq-tv-z]|[nN][pP]?', Keyword, 'statement')
+        ],
+
+        'single-line string': [
+            (r'"', String, '#pop'),
+            (r'[^\\"]+', String)
+        ]
+    }
+
+
+class CsoundOrchestraLexer(CsoundLexer):
+    """
+    For `Csound <http://csound.github.io>`_ orchestras.
+
+    .. versionadded:: 2.1
+    """
+
+    name = 'Csound Orchestra'
+    aliases = ['csound', 'csound-orc']
+    filenames = ['*.orc']
+
+    user_defined_opcodes = set()
+
+    def opcode_name_callback(lexer, match):
+        opcode = match.group(0)
+        lexer.user_defined_opcodes.add(opcode)
+        yield match.start(), Name.Function, opcode
+
+    def name_callback(lexer, match):
+        name = match.group(0)
+        if re.match('p\d+$', name) or name in OPCODES:
+            yield match.start(), Name.Builtin, name
+        elif name in lexer.user_defined_opcodes:
+            yield match.start(), Name.Function, name
+        else:
+            nameMatch = re.search(r'^(g?[aikSw])(\w+)', name)
+            if nameMatch:
+                yield nameMatch.start(1), Keyword.Type, nameMatch.group(1)
+                yield nameMatch.start(2), Name, nameMatch.group(2)
+            else:
+                yield match.start(), Name, name
+
+    tokens = {
+        'label': [
+            (r'\b(\w+)(:)', bygroups(Name.Label, Punctuation))
+        ],
+
+        'partial expression': [
+            include('preprocessor directives'),
+            (r'\b(0dbfs|k(r|smps)|nchnls(_i)?|sr)\b', Name.Variable.Global),
+            (r'\d+e[+-]?\d+|(\d+\.\d*|\d*\.\d+)(e[+-]?\d+)?', Number.Float),
+            (r'0[xX][a-fA-F0-9]+', Number.Hex),
+            (r'\d+', Number.Integer),
+            (r'"', String, 'single-line string'),
+            (r'\{\{', String, 'multi-line string'),
+            (r'[+\-*/%^!=&|<>#~¬]', Operator),
+            (r'[](),?:[]', Punctuation),
+            (words((
+                # Keywords
+                'do', 'else', 'elseif', 'endif', 'enduntil', 'fi', 'if', 'ithen', 'kthen',
+                'od', 'then', 'until', 'while',
+                # Opcodes that act as control structures
+                'return', 'timout'
+                ), prefix=r'\b', suffix=r'\b'), Keyword),
+            (words(('goto', 'igoto', 'kgoto', 'rigoto', 'tigoto'),
+                   prefix=r'\b', suffix=r'\b'), Keyword, 'goto label'),
+            (words(('cggoto', 'cigoto', 'cingoto', 'ckgoto', 'cngoto'),
+                   prefix=r'\b', suffix=r'\b'), Keyword,
+             ('goto label', 'goto expression')),
+            (words(('loop_ge', 'loop_gt', 'loop_le', 'loop_lt'),
+                   prefix=r'\b', suffix=r'\b'), Keyword,
+             ('goto label', 'goto expression', 'goto expression', 'goto expression')),
+            (r'\bscoreline(_i)?\b', Name.Builtin, 'scoreline opcode'),
+            (r'\bpyl?run[it]?\b', Name.Builtin, 'python opcode'),
+            (r'\blua_(exec|opdef)\b', Name.Builtin, 'lua opcode'),
+            (r'\b[a-zA-Z_]\w*\b', name_callback)
+        ],
+
+        'expression': [
+            include('whitespace or macro call'),
+            newline + ('#pop',),
+            include('partial expression')
+        ],
+
+        'root': [
+            newline,
+            include('whitespace or macro call'),
+            (r'\binstr\b', Keyword, ('instrument block', 'instrument name list')),
+            (r'\bopcode\b', Keyword, ('opcode block', 'opcode parameter list',
+                                      'opcode types', 'opcode types', 'opcode name')),
+            include('label'),
+            default('expression')
+        ],
+
+        'instrument name list': [
+            include('whitespace or macro call'),
+            (r'\d+|\+?[a-zA-Z_]\w*', Name.Function),
+            (r',', Punctuation),
+            newline + ('#pop',)
+        ],
+        'instrument block': [
+            newline,
+            include('whitespace or macro call'),
+            (r'\bendin\b', Keyword, '#pop'),
+            include('label'),
+            default('expression')
+        ],
+
+        'opcode name': [
+            include('whitespace or macro call'),
+            (r'[a-zA-Z_]\w*', opcode_name_callback, '#pop')
+        ],
+        'opcode types': [
+            include('whitespace or macro call'),
+            (r'0|[]afijkKoOpPStV[]+', Keyword.Type, '#pop'),
+            (r',', Punctuation)
+        ],
+        'opcode parameter list': [
+            include('whitespace or macro call'),
+            newline + ('#pop',)
+        ],
+        'opcode block': [
+            newline,
+            include('whitespace or macro call'),
+            (r'\bendop\b', Keyword, '#pop'),
+            include('label'),
+            default('expression')
+        ],
+
+        'goto label': [
+            include('whitespace or macro call'),
+            (r'\w+', Name.Label, '#pop'),
+            default('#pop')
+        ],
+        'goto expression': [
+            include('whitespace or macro call'),
+            (r',', Punctuation, '#pop'),
+            include('partial expression')
+        ],
+
+        'single-line string': [
+            include('macro call'),
+            (r'"', String, '#pop'),
+            # From https://github.com/csound/csound/blob/develop/Opcodes/fout.c#L1405
+            (r'%\d*(\.\d+)?[cdhilouxX]', String.Interpol),
+            (r'%[!%nNrRtT]|[~^]|\\([\\aAbBnNrRtT"]|[0-7]{1,3})', String.Escape),
+            (r'[^\\"~$%\^\n]+', String),
+            (r'[\\"~$%\^\n]', String)
+        ],
+        'multi-line string': [
+            (r'\}\}', String, '#pop'),
+            (r'[^}]+|\}(?!\})', String)
+        ],
+
+        'scoreline opcode': [
+            include('whitespace or macro call'),
+            (r'\{\{', String, 'scoreline'),
+            default('#pop')
+        ],
+        'scoreline': [
+            (r'\}\}', String, '#pop'),
+            (r'([^}]+)|\}(?!\})', using(CsoundScoreLexer))
+        ],
+
+        'python opcode': [
+            include('whitespace or macro call'),
+            (r'\{\{', String, 'python'),
+            default('#pop')
+        ],
+        'python': [
+            (r'\}\}', String, '#pop'),
+            (r'([^}]+)|\}(?!\})', using(PythonLexer))
+        ],
+
+        'lua opcode': [
+            include('whitespace or macro call'),
+            (r'"', String, 'single-line string'),
+            (r'\{\{', String, 'lua'),
+            (r',', Punctuation),
+            default('#pop')
+        ],
+        'lua': [
+            (r'\}\}', String, '#pop'),
+            (r'([^}]+)|\}(?!\})', using(LuaLexer))
+        ]
+    }
+
+
+class CsoundDocumentLexer(RegexLexer):
+    """
+    For `Csound <http://csound.github.io>`_ documents.
+
+    .. versionadded:: 2.1
+    """
+
+    name = 'Csound Document'
+    aliases = ['csound-document', 'csound-csd']
+    filenames = ['*.csd']
+
+    # These tokens are based on those in XmlLexer in pygments/lexers/html.py. Making
+    # CsoundDocumentLexer a subclass of XmlLexer rather than RegexLexer may seem like a
+    # better idea, since Csound Document files look like XML files. However, Csound
+    # Documents can contain Csound comments (preceded by //, for example) before and
+    # after the root element, unescaped bitwise AND & and less than < operators, etc. In
+    # other words, while Csound Document files look like XML files, they may not actually
+    # be XML files.
+    tokens = {
+        'root': [
+            newline,
+            (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+            (r'[^<&;/]+', Text),
+            (r'<\s*CsInstruments', Name.Tag, ('orchestra', 'tag')),
+            (r'<\s*CsScore', Name.Tag, ('score', 'tag')),
+            (r'<\s*[hH][tT][mM][lL]', Name.Tag, ('HTML', 'tag')),
+            (r'<\s*[\w:.-]+', Name.Tag, 'tag'),
+            (r'<\s*/\s*[\w:.-]+\s*>', Name.Tag)
+        ],
+        'orchestra': [
+            (r'<\s*/\s*CsInstruments\s*>', Name.Tag, '#pop'),
+            (r'(.|\n)+?(?=<\s*/\s*CsInstruments\s*>)', using(CsoundOrchestraLexer))
+        ],
+        'score': [
+            (r'<\s*/\s*CsScore\s*>', Name.Tag, '#pop'),
+            (r'(.|\n)+?(?=<\s*/\s*CsScore\s*>)', using(CsoundScoreLexer))
+        ],
+        'HTML': [
+            (r'<\s*/\s*[hH][tT][mM][lL]\s*>', Name.Tag, '#pop'),
+            (r'(.|\n)+?(?=<\s*/\s*[hH][tT][mM][lL]\s*>)', using(HtmlLexer))
+        ],
+        'tag': [
+            (r'\s+', Text),
+            (r'[\w.:-]+\s*=', Name.Attribute, 'attr'),
+            (r'/?\s*>', Name.Tag, '#pop')
+        ],
+        'attr': [
+            (r'\s+', Text),
+            (r'".*?"', String, '#pop'),
+            (r"'.*?'", String, '#pop'),
+            (r'[^\s>]+', String, '#pop')
+        ]
+    }

packages/wakatime/packages/pygments/lexers/css.py

@@ -0,0 +1,689 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.css
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for CSS and related stylesheet formats.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+import copy
+
+from pygments.lexer import ExtendedRegexLexer, RegexLexer, include, bygroups, \
+    default, words, inherit
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+from pygments.util import iteritems
+
+__all__ = ['CssLexer', 'SassLexer', 'ScssLexer', 'LessCssLexer']
+
+
+# List of vendor prefixes obtained from:
+# https://www.w3.org/TR/CSS21/syndata.html#vendor-keyword-history
+_vendor_prefixes = (
+    '-ms-', 'mso-', '-moz-', '-o-', '-xv-', '-atsc-', '-wap-', '-khtml-',
+    '-webkit-', 'prince-', '-ah-', '-hp-', '-ro-', '-rim-', '-tc-',
+)
+
+# List of CSS properties obtained from:
+# https://www.w3.org/Style/CSS/all-properties.en.html
+# Note: handle --* separately
+_css_properties = (
+    'align-content', 'align-items', 'align-self', 'alignment-baseline', 'all',
+    'animation', 'animation-delay', 'animation-direction',
+    'animation-duration', 'animation-fill-mode', 'animation-iteration-count',
+    'animation-name', 'animation-play-state', 'animation-timing-function',
+    'appearance', 'azimuth', 'backface-visibility', 'background',
+    'background-attachment', 'background-blend-mode', 'background-clip',
+    'background-color', 'background-image', 'background-origin',
+    'background-position', 'background-repeat', 'background-size',
+    'baseline-shift', 'bookmark-label', 'bookmark-level', 'bookmark-state',
+    'border', 'border-bottom', 'border-bottom-color',
+    'border-bottom-left-radius', 'border-bottom-right-radius',
+    'border-bottom-style', 'border-bottom-width', 'border-boundary',
+    'border-collapse', 'border-color', 'border-image', 'border-image-outset',
+    'border-image-repeat', 'border-image-slice', 'border-image-source',
+    'border-image-width', 'border-left', 'border-left-color',
+    'border-left-style', 'border-left-width', 'border-radius', 'border-right',
+    'border-right-color', 'border-right-style', 'border-right-width',
+    'border-spacing', 'border-style', 'border-top', 'border-top-color',
+    'border-top-left-radius', 'border-top-right-radius', 'border-top-style',
+    'border-top-width', 'border-width', 'bottom', 'box-decoration-break',
+    'box-shadow', 'box-sizing', 'box-snap', 'box-suppress', 'break-after',
+    'break-before', 'break-inside', 'caption-side', 'caret', 'caret-animation',
+    'caret-color', 'caret-shape', 'chains', 'clear', 'clip', 'clip-path',
+    'clip-rule', 'color', 'color-interpolation-filters', 'column-count',
+    'column-fill', 'column-gap', 'column-rule', 'column-rule-color',
+    'column-rule-style', 'column-rule-width', 'column-span', 'column-width',
+    'columns', 'content', 'counter-increment', 'counter-reset', 'counter-set',
+    'crop', 'cue', 'cue-after', 'cue-before', 'cursor', 'direction', 'display',
+    'dominant-baseline', 'elevation', 'empty-cells', 'filter', 'flex',
+    'flex-basis', 'flex-direction', 'flex-flow', 'flex-grow', 'flex-shrink',
+    'flex-wrap', 'float', 'float-defer', 'float-offset', 'float-reference',
+    'flood-color', 'flood-opacity', 'flow', 'flow-from', 'flow-into', 'font',
+    'font-family', 'font-feature-settings', 'font-kerning',
+    'font-language-override', 'font-size', 'font-size-adjust', 'font-stretch',
+    'font-style', 'font-synthesis', 'font-variant', 'font-variant-alternates',
+    'font-variant-caps', 'font-variant-east-asian', 'font-variant-ligatures',
+    'font-variant-numeric', 'font-variant-position', 'font-weight',
+    'footnote-display', 'footnote-policy', 'glyph-orientation-vertical',
+    'grid', 'grid-area', 'grid-auto-columns', 'grid-auto-flow',
+    'grid-auto-rows', 'grid-column', 'grid-column-end', 'grid-column-gap',
+    'grid-column-start', 'grid-gap', 'grid-row', 'grid-row-end',
+    'grid-row-gap', 'grid-row-start', 'grid-template', 'grid-template-areas',
+    'grid-template-columns', 'grid-template-rows', 'hanging-punctuation',
+    'height', 'hyphenate-character', 'hyphenate-limit-chars',
+    'hyphenate-limit-last', 'hyphenate-limit-lines', 'hyphenate-limit-zone',
+    'hyphens', 'image-orientation', 'image-resolution', 'initial-letter',
+    'initial-letter-align', 'initial-letter-wrap', 'isolation',
+    'justify-content', 'justify-items', 'justify-self', 'left',
+    'letter-spacing', 'lighting-color', 'line-break', 'line-grid',
+    'line-height', 'line-snap', 'list-style', 'list-style-image',
+    'list-style-position', 'list-style-type', 'margin', 'margin-bottom',
+    'margin-left', 'margin-right', 'margin-top', 'marker-side',
+    'marquee-direction', 'marquee-loop', 'marquee-speed', 'marquee-style',
+    'mask', 'mask-border', 'mask-border-mode', 'mask-border-outset',
+    'mask-border-repeat', 'mask-border-slice', 'mask-border-source',
+    'mask-border-width', 'mask-clip', 'mask-composite', 'mask-image',
+    'mask-mode', 'mask-origin', 'mask-position', 'mask-repeat', 'mask-size',
+    'mask-type', 'max-height', 'max-lines', 'max-width', 'min-height',
+    'min-width', 'mix-blend-mode', 'motion', 'motion-offset', 'motion-path',
+    'motion-rotation', 'move-to', 'nav-down', 'nav-left', 'nav-right',
+    'nav-up', 'object-fit', 'object-position', 'offset-after', 'offset-before',
+    'offset-end', 'offset-start', 'opacity', 'order', 'orphans', 'outline',
+    'outline-color', 'outline-offset', 'outline-style', 'outline-width',
+    'overflow', 'overflow-style', 'overflow-wrap', 'overflow-x', 'overflow-y',
+    'padding', 'padding-bottom', 'padding-left', 'padding-right', 'padding-top',
+    'page', 'page-break-after', 'page-break-before', 'page-break-inside',
+    'page-policy', 'pause', 'pause-after', 'pause-before', 'perspective',
+    'perspective-origin', 'pitch', 'pitch-range', 'play-during', 'polar-angle',
+    'polar-distance', 'position', 'presentation-level', 'quotes',
+    'region-fragment', 'resize', 'rest', 'rest-after', 'rest-before',
+    'richness', 'right', 'rotation', 'rotation-point', 'ruby-align',
+    'ruby-merge', 'ruby-position', 'running', 'scroll-snap-coordinate',
+    'scroll-snap-destination', 'scroll-snap-points-x', 'scroll-snap-points-y',
+    'scroll-snap-type', 'shape-image-threshold', 'shape-inside', 'shape-margin',
+    'shape-outside', 'size', 'speak', 'speak-as', 'speak-header',
+    'speak-numeral', 'speak-punctuation', 'speech-rate', 'stress', 'string-set',
+    'tab-size', 'table-layout', 'text-align', 'text-align-last',
+    'text-combine-upright', 'text-decoration', 'text-decoration-color',
+    'text-decoration-line', 'text-decoration-skip', 'text-decoration-style',
+    'text-emphasis', 'text-emphasis-color', 'text-emphasis-position',
+    'text-emphasis-style', 'text-indent', 'text-justify', 'text-orientation',
+    'text-overflow', 'text-shadow', 'text-space-collapse', 'text-space-trim',
+    'text-spacing', 'text-transform', 'text-underline-position', 'text-wrap',
+    'top', 'transform', 'transform-origin', 'transform-style', 'transition',
+    'transition-delay', 'transition-duration', 'transition-property',
+    'transition-timing-function', 'unicode-bidi', 'user-select',
+    'vertical-align', 'visibility', 'voice-balance', 'voice-duration',
+    'voice-family', 'voice-pitch', 'voice-range', 'voice-rate', 'voice-stress',
+    'voice-volume', 'volume', 'white-space', 'widows', 'width', 'will-change',
+    'word-break', 'word-spacing', 'word-wrap', 'wrap-after', 'wrap-before',
+    'wrap-flow', 'wrap-inside', 'wrap-through', 'writing-mode', 'z-index',
+)
+
+# List of keyword values obtained from: 
+# http://cssvalues.com/
+_keyword_values = (
+    'absolute', 'alias', 'all', 'all-petite-caps', 'all-scroll',
+    'all-small-caps', 'allow-end', 'alpha', 'alternate', 'alternate-reverse',
+    'always', 'armenian', 'auto', 'avoid', 'avoid-column', 'avoid-page',
+    'backwards', 'balance', 'baseline', 'below', 'blink', 'block', 'bold',
+    'bolder', 'border-box', 'both', 'bottom', 'box-decoration', 'break-word',
+    'capitalize', 'cell', 'center', 'circle', 'clip', 'clone', 'close-quote',
+    'col-resize', 'collapse', 'color', 'color-burn', 'color-dodge', 'column',
+    'column-reverse', 'compact', 'condensed', 'contain', 'container',
+    'content-box', 'context-menu', 'copy', 'cover', 'crisp-edges', 'crosshair',
+    'currentColor', 'cursive', 'darken', 'dashed', 'decimal',
+    'decimal-leading-zero', 'default', 'descendants', 'difference', 'digits',
+    'disc', 'distribute', 'dot', 'dotted', 'double', 'double-circle', 'e-resize',
+    'each-line', 'ease', 'ease-in', 'ease-in-out', 'ease-out', 'edges',
+    'ellipsis', 'end', 'ew-resize', 'exclusion', 'expanded', 'extra-condensed',
+    'extra-expanded', 'fantasy', 'fill', 'fill-box', 'filled', 'first', 'fixed',
+    'flat', 'flex', 'flex-end', 'flex-start', 'flip', 'force-end', 'forwards',
+    'from-image', 'full-width', 'geometricPrecision', 'georgian', 'groove',
+    'hanging', 'hard-light', 'help', 'hidden', 'hide', 'horizontal', 'hue',
+    'icon', 'infinite', 'inherit', 'initial', 'ink', 'inline', 'inline-block',
+    'inline-flex', 'inline-table', 'inset', 'inside', 'inter-word', 'invert',
+    'isolate', 'italic', 'justify', 'large', 'larger', 'last', 'left',
+    'lighten', 'lighter', 'line-through', 'linear', 'list-item', 'local',
+    'loose', 'lower-alpha', 'lower-greek', 'lower-latin', 'lower-roman',
+    'lowercase', 'ltr', 'luminance', 'luminosity', 'mandatory', 'manipulation',
+    'manual', 'margin-box', 'match-parent', 'medium', 'mixed', 'monospace',
+    'move', 'multiply', 'n-resize', 'ne-resize', 'nesw-resize',
+    'no-close-quote', 'no-drop', 'no-open-quote', 'no-repeat', 'none', 'normal',
+    'not-allowed', 'nowrap', 'ns-resize', 'nw-resize', 'nwse-resize', 'objects',
+    'oblique', 'off', 'on', 'open', 'open-quote', 'optimizeLegibility',
+    'optimizeSpeed', 'outset', 'outside', 'over', 'overlay', 'overline',
+    'padding-box', 'page', 'pan-down', 'pan-left', 'pan-right', 'pan-up',
+    'pan-x', 'pan-y', 'paused', 'petite-caps', 'pixelated', 'pointer',
+    'preserve-3d', 'progress', 'proximity', 'relative', 'repeat',
+    'repeat no-repeat', 'repeat-x', 'repeat-y', 'reverse', 'ridge', 'right',
+    'round', 'row', 'row-resize', 'row-reverse', 'rtl', 'ruby', 'ruby-base',
+    'ruby-base-container', 'ruby-text', 'ruby-text-container', 'run-in',
+    'running', 's-resize', 'sans-serif', 'saturation', 'scale-down', 'screen',
+    'scroll', 'se-resize', 'semi-condensed', 'semi-expanded', 'separate',
+    'serif', 'sesame', 'show', 'sideways', 'sideways-left', 'sideways-right',
+    'slice', 'small', 'small-caps', 'smaller', 'smooth', 'snap', 'soft-light',
+    'solid', 'space', 'space-around', 'space-between', 'spaces', 'square',
+    'start', 'static', 'step-end', 'step-start', 'sticky', 'stretch', 'strict',
+    'stroke-box', 'style', 'sw-resize', 'table', 'table-caption', 'table-cell',
+    'table-column', 'table-column-group', 'table-footer-group',
+    'table-header-group', 'table-row', 'table-row-group', 'text', 'thick',
+    'thin', 'titling-caps', 'to', 'top', 'triangle', 'ultra-condensed',
+    'ultra-expanded', 'under', 'underline', 'unicase', 'unset', 'upper-alpha',
+    'upper-latin', 'upper-roman', 'uppercase', 'upright', 'use-glyph-orientation',
+    'vertical', 'vertical-text', 'view-box', 'visible', 'w-resize', 'wait',
+    'wavy', 'weight', 'weight style', 'wrap', 'wrap-reverse', 'x-large',
+    'x-small', 'xx-large', 'xx-small', 'zoom-in', 'zoom-out',
+)
+
+# List of extended color keywords obtained from:
+# https://drafts.csswg.org/css-color/#named-colors
+_color_keywords = (
+    'aliceblue', 'antiquewhite', 'aqua', 'aquamarine', 'azure', 'beige',
+    'bisque', 'black', 'blanchedalmond', 'blue', 'blueviolet', 'brown',
+    'burlywood', 'cadetblue', 'chartreuse', 'chocolate', 'coral',
+    'cornflowerblue', 'cornsilk', 'crimson', 'cyan', 'darkblue', 'darkcyan',
+    'darkgoldenrod', 'darkgray', 'darkgreen', 'darkgrey', 'darkkhaki',
+    'darkmagenta', 'darkolivegreen', 'darkorange', 'darkorchid', 'darkred',
+    'darksalmon', 'darkseagreen', 'darkslateblue', 'darkslategray',
+    'darkslategrey', 'darkturquoise', 'darkviolet', 'deeppink', 'deepskyblue',
+    'dimgray', 'dimgrey', 'dodgerblue', 'firebrick', 'floralwhite',
+    'forestgreen', 'fuchsia', 'gainsboro', 'ghostwhite', 'gold', 'goldenrod',
+    'gray', 'green', 'greenyellow', 'grey', 'honeydew', 'hotpink', 'indianred',
+    'indigo', 'ivory', 'khaki', 'lavender', 'lavenderblush', 'lawngreen',
+    'lemonchiffon', 'lightblue', 'lightcoral', 'lightcyan',
+    'lightgoldenrodyellow', 'lightgray', 'lightgreen', 'lightgrey',
+    'lightpink', 'lightsalmon', 'lightseagreen', 'lightskyblue',
+    'lightslategray', 'lightslategrey', 'lightsteelblue', 'lightyellow',
+    'lime', 'limegreen', 'linen', 'magenta', 'maroon', 'mediumaquamarine',
+    'mediumblue', 'mediumorchid', 'mediumpurple', 'mediumseagreen',
+    'mediumslateblue', 'mediumspringgreen', 'mediumturquoise',
+    'mediumvioletred', 'midnightblue', 'mintcream', 'mistyrose', 'moccasin',
+    'navajowhite', 'navy', 'oldlace', 'olive', 'olivedrab', 'orange',
+    'orangered', 'orchid', 'palegoldenrod', 'palegreen', 'paleturquoise',
+    'palevioletred', 'papayawhip', 'peachpuff', 'peru', 'pink', 'plum',
+    'powderblue', 'purple', 'rebeccapurple', 'red', 'rosybrown', 'royalblue',
+    'saddlebrown', 'salmon', 'sandybrown', 'seagreen', 'seashell', 'sienna',
+    'silver', 'skyblue', 'slateblue', 'slategray', 'slategrey', 'snow',
+    'springgreen', 'steelblue', 'tan', 'teal', 'thistle', 'tomato', 'turquoise',
+    'violet', 'wheat', 'white', 'whitesmoke', 'yellow', 'yellowgreen',
+) + ('transparent',)
+
+# List of other keyword values from other sources:
+_other_keyword_values = (
+    'above', 'aural', 'behind', 'bidi-override', 'center-left', 'center-right',
+    'cjk-ideographic', 'continuous', 'crop', 'cross', 'embed', 'far-left',
+    'far-right', 'fast', 'faster', 'hebrew', 'high', 'higher', 'hiragana',
+    'hiragana-iroha', 'katakana', 'katakana-iroha', 'landscape', 'left-side',
+    'leftwards', 'level', 'loud', 'low', 'lower', 'message-box', 'middle',
+    'mix', 'narrower', 'once', 'portrait', 'right-side', 'rightwards', 'silent',
+    'slow', 'slower', 'small-caption', 'soft', 'spell-out', 'status-bar',
+    'super', 'text-bottom', 'text-top', 'wider', 'x-fast', 'x-high', 'x-loud',
+    'x-low', 'x-soft', 'yes', 'pre', 'pre-wrap', 'pre-line',
+)
+
+# List of functional notation and function keyword values:
+_functional_notation_keyword_values = (
+    'attr', 'blackness', 'blend', 'blenda', 'blur', 'brightness', 'calc',
+    'circle', 'color-mod', 'contrast', 'counter', 'cubic-bezier', 'device-cmyk',
+    'drop-shadow', 'ellipse', 'gray', 'grayscale', 'hsl', 'hsla', 'hue',
+    'hue-rotate', 'hwb', 'image', 'inset', 'invert', 'lightness',
+    'linear-gradient', 'matrix', 'matrix3d', 'opacity', 'perspective',
+    'polygon', 'radial-gradient', 'rect', 'repeating-linear-gradient',
+    'repeating-radial-gradient', 'rgb', 'rgba', 'rotate', 'rotate3d', 'rotateX',
+    'rotateY', 'rotateZ', 'saturate', 'saturation', 'scale', 'scale3d',
+    'scaleX', 'scaleY', 'scaleZ', 'sepia', 'shade', 'skewX', 'skewY', 'steps',
+    'tint', 'toggle', 'translate', 'translate3d', 'translateX', 'translateY',
+    'translateZ', 'whiteness',
+)
+# Note! Handle url(...) separately.
+
+# List of units obtained from:
+# https://www.w3.org/TR/css3-values/
+_angle_units = (
+    'deg', 'grad', 'rad', 'turn',
+)
+_frequency_units = (
+    'Hz', 'kHz',
+)
+_length_units = (
+    'em', 'ex', 'ch', 'rem',
+    'vh', 'vw', 'vmin', 'vmax',
+    'px', 'mm', 'cm', 'in', 'pt', 'pc', 'q',
+)
+_resolution_units = (
+    'dpi', 'dpcm', 'dppx',
+)
+_time_units = (
+    's', 'ms',
+)
+_all_units = _angle_units + _frequency_units + _length_units + \
+             _resolution_units + _time_units
+
+
+class CssLexer(RegexLexer):
+    """
+    For CSS (Cascading Style Sheets).
+    """
+
+    name = 'CSS'
+    aliases = ['css']
+    filenames = ['*.css']
+    mimetypes = ['text/css']
+
+    tokens = {
+        'root': [
+            include('basics'),
+        ],
+        'basics': [
+            (r'\s+', Text),
+            (r'/\*(?:.|\n)*?\*/', Comment),
+            (r'\{', Punctuation, 'content'),
+            (r'(\:{1,2})([\w-]+)', bygroups(Punctuation, Name.Decorator)),
+            (r'(\.)([\w-]+)', bygroups(Punctuation, Name.Class)),
+            (r'(\#)([\w-]+)', bygroups(Punctuation, Name.Namespace)),
+            (r'(@)([\w-]+)', bygroups(Punctuation, Keyword), 'atrule'),
+            (r'[\w-]+', Name.Tag),
+            (r'[~^*!%&$\[\]()<>|+=@:;,./?-]', Operator),
+            (r'"(\\\\|\\"|[^"])*"', String.Double),
+            (r"'(\\\\|\\'|[^'])*'", String.Single)
+        ],
+        'atrule': [
+            (r'\{', Punctuation, 'atcontent'),
+            (r';', Punctuation, '#pop'),
+            include('basics'),
+        ],
+        'atcontent': [
+            include('basics'),
+            (r'\}', Punctuation, '#pop:2'),
+        ],
+        'content': [
+            (r'\s+', Text),
+            (r'\}', Punctuation, '#pop'),
+            (r';', Punctuation),
+            (r'^@.*?$', Comment.Preproc),
+
+            (words(_vendor_prefixes,), Keyword.Pseudo),
+            (r'('+r'|'.join(_css_properties)+r')(\s*)(\:)',
+             bygroups(Keyword, Text, Punctuation), 'value-start'),
+            (r'([a-zA-Z_][\w-]*)(\s*)(\:)', bygroups(Name, Text, Punctuation),
+             'value-start'),
+
+            (r'/\*(?:.|\n)*?\*/', Comment),
+        ],
+        'value-start': [
+            (r'\s+', Text),
+            (words(_vendor_prefixes,), Name.Builtin.Pseudo),
+            include('urls'),
+            (r'('+r'|'.join(_functional_notation_keyword_values)+r')(\()',
+             bygroups(Name.Builtin, Punctuation), 'function-start'),
+            (r'([a-zA-Z_][\w-]+)(\()', bygroups(Name.Function, Punctuation), 'function-start'),
+            (words(_keyword_values, suffix=r'\b'), Keyword.Constant),
+            (words(_other_keyword_values, suffix=r'\b'), Keyword.Constant),
+            (words(_color_keywords, suffix=r'\b'), Keyword.Constant),
+            (words(_css_properties, suffix=r'\b'), Keyword), # for transition-property etc.
+            (r'\!important', Comment.Preproc),
+            (r'/\*(?:.|\n)*?\*/', Comment),
+
+            include('numeric-values'),
+            
+            (r'[~^*!%&<>|+=@:./?-]+', Operator),
+            (r'[\[\](),]+', Punctuation),
+            (r'"(\\\\|\\"|[^"])*"', String.Double),
+            (r"'(\\\\|\\'|[^'])*'", String.Single),
+            (r'[a-zA-Z_][\w-]*', Name),
+            (r';', Punctuation, '#pop'),
+            (r'\}', Punctuation, '#pop:2'),
+        ],
+        'function-start': [
+            (r'\s+', Text),
+            include('urls'),
+            (words(_vendor_prefixes,), Keyword.Pseudo),
+            (words(_keyword_values, suffix=r'\b'), Keyword.Constant),
+            (words(_other_keyword_values, suffix=r'\b'), Keyword.Constant),
+            (words(_color_keywords, suffix=r'\b'), Keyword.Constant),
+
+            # function-start may be entered recursively
+            (r'(' + r'|'.join(_functional_notation_keyword_values) + r')(\()',
+             bygroups(Name.Builtin, Punctuation), 'function-start'),
+            (r'([a-zA-Z_][\w-]+)(\()', bygroups(Name.Function, Punctuation), 'function-start'),
+
+            (r'/\*(?:.|\n)*?\*/', Comment),
+            include('numeric-values'),
+            (r'[*+/-]', Operator),
+            (r'[,]', Punctuation),
+            (r'"(\\\\|\\"|[^"])*"', String.Double),
+            (r"'(\\\\|\\'|[^'])*'", String.Single),
+            (r'[a-zA-Z_-]\w*', Name),
+            (r'\)', Punctuation, '#pop'),
+        ],
+        'urls': [
+            (r'(url)(\()(".*?")(\))', bygroups(Name.Builtin, Punctuation,
+                                               String.Double, Punctuation)),
+            (r"(url)(\()('.*?')(\))", bygroups(Name.Builtin, Punctuation,
+                                               String.Single, Punctuation)),
+            (r'(url)(\()(.*?)(\))', bygroups(Name.Builtin, Punctuation,
+                                             String.Other, Punctuation)),
+        ],
+        'numeric-values': [
+            (r'\#[a-zA-Z0-9]{1,6}', Number.Hex),
+            (r'[+\-]?[0-9]*[.][0-9]+', Number.Float, 'numeric-end'),
+            (r'[+\-]?[0-9]+', Number.Integer, 'numeric-end'),            
+        ],        
+        'numeric-end': [
+            (words(_all_units, suffix=r'\b'), Keyword.Type),
+            (r'%', Keyword.Type),
+            default('#pop'),
+        ],
+    }
+
+
+common_sass_tokens = {
+    'value': [
+        (r'[ \t]+', Text),
+        (r'[!$][\w-]+', Name.Variable),
+        (r'url\(', String.Other, 'string-url'),
+        (r'[a-z_-][\w-]*(?=\()', Name.Function),
+        (words(_css_properties + (
+            'above', 'absolute', 'always', 'armenian', 'aural', 'auto', 'avoid', 'baseline',
+            'behind', 'below', 'bidi-override', 'blink', 'block', 'bold', 'bolder', 'both',
+            'capitalize', 'center-left', 'center-right', 'center', 'circle',
+            'cjk-ideographic', 'close-quote', 'collapse', 'condensed', 'continuous',
+            'crop', 'crosshair', 'cross', 'cursive', 'dashed', 'decimal-leading-zero',
+            'decimal', 'default', 'digits', 'disc', 'dotted', 'double', 'e-resize', 'embed',
+            'extra-condensed', 'extra-expanded', 'expanded', 'fantasy', 'far-left',
+            'far-right', 'faster', 'fast', 'fixed', 'georgian', 'groove', 'hebrew', 'help',
+            'hidden', 'hide', 'higher', 'high', 'hiragana-iroha', 'hiragana', 'icon',
+            'inherit', 'inline-table', 'inline', 'inset', 'inside', 'invert', 'italic',
+            'justify', 'katakana-iroha', 'katakana', 'landscape', 'larger', 'large',
+            'left-side', 'leftwards', 'level', 'lighter', 'line-through', 'list-item',
+            'loud', 'lower-alpha', 'lower-greek', 'lower-roman', 'lowercase', 'ltr',
+            'lower', 'low', 'medium', 'message-box', 'middle', 'mix', 'monospace',
+            'n-resize', 'narrower', 'ne-resize', 'no-close-quote', 'no-open-quote',
+            'no-repeat', 'none', 'normal', 'nowrap', 'nw-resize', 'oblique', 'once',
+            'open-quote', 'outset', 'outside', 'overline', 'pointer', 'portrait', 'px',
+            'relative', 'repeat-x', 'repeat-y', 'repeat', 'rgb', 'ridge', 'right-side',
+            'rightwards', 's-resize', 'sans-serif', 'scroll', 'se-resize',
+            'semi-condensed', 'semi-expanded', 'separate', 'serif', 'show', 'silent',
+            'slow', 'slower', 'small-caps', 'small-caption', 'smaller', 'soft', 'solid',
+            'spell-out', 'square', 'static', 'status-bar', 'super', 'sw-resize',
+            'table-caption', 'table-cell', 'table-column', 'table-column-group',
+            'table-footer-group', 'table-header-group', 'table-row',
+            'table-row-group', 'text', 'text-bottom', 'text-top', 'thick', 'thin',
+            'transparent', 'ultra-condensed', 'ultra-expanded', 'underline',
+            'upper-alpha', 'upper-latin', 'upper-roman', 'uppercase', 'url',
+            'visible', 'w-resize', 'wait', 'wider', 'x-fast', 'x-high', 'x-large', 'x-loud',
+            'x-low', 'x-small', 'x-soft', 'xx-large', 'xx-small', 'yes'), suffix=r'\b'),
+         Name.Constant),
+        (words(_color_keywords, suffix=r'\b'), Name.Entity),
+        (words((
+            'black', 'silver', 'gray', 'white', 'maroon', 'red', 'purple', 'fuchsia', 'green',
+            'lime', 'olive', 'yellow', 'navy', 'blue', 'teal', 'aqua'), suffix=r'\b'),
+         Name.Builtin),
+        (r'\!(important|default)', Name.Exception),
+        (r'(true|false)', Name.Pseudo),
+        (r'(and|or|not)', Operator.Word),
+        (r'/\*', Comment.Multiline, 'inline-comment'),
+        (r'//[^\n]*', Comment.Single),
+        (r'\#[a-z0-9]{1,6}', Number.Hex),
+        (r'(-?\d+)(\%|[a-z]+)?', bygroups(Number.Integer, Keyword.Type)),
+        (r'(-?\d*\.\d+)(\%|[a-z]+)?', bygroups(Number.Float, Keyword.Type)),
+        (r'#\{', String.Interpol, 'interpolation'),
+        (r'[~^*!&%<>|+=@:,./?-]+', Operator),
+        (r'[\[\]()]+', Punctuation),
+        (r'"', String.Double, 'string-double'),
+        (r"'", String.Single, 'string-single'),
+        (r'[a-z_-][\w-]*', Name),
+    ],
+
+    'interpolation': [
+        (r'\}', String.Interpol, '#pop'),
+        include('value'),
+    ],
+
+    'selector': [
+        (r'[ \t]+', Text),
+        (r'\:', Name.Decorator, 'pseudo-class'),
+        (r'\.', Name.Class, 'class'),
+        (r'\#', Name.Namespace, 'id'),
+        (r'[\w-]+', Name.Tag),
+        (r'#\{', String.Interpol, 'interpolation'),
+        (r'&', Keyword),
+        (r'[~^*!&\[\]()<>|+=@:;,./?-]', Operator),
+        (r'"', String.Double, 'string-double'),
+        (r"'", String.Single, 'string-single'),
+    ],
+
+    'string-double': [
+        (r'(\\.|#(?=[^\n{])|[^\n"#])+', String.Double),
+        (r'#\{', String.Interpol, 'interpolation'),
+        (r'"', String.Double, '#pop'),
+    ],
+
+    'string-single': [
+        (r"(\\.|#(?=[^\n{])|[^\n'#])+", String.Double),
+        (r'#\{', String.Interpol, 'interpolation'),
+        (r"'", String.Double, '#pop'),
+    ],
+
+    'string-url': [
+        (r'(\\#|#(?=[^\n{])|[^\n#)])+', String.Other),
+        (r'#\{', String.Interpol, 'interpolation'),
+        (r'\)', String.Other, '#pop'),
+    ],
+
+    'pseudo-class': [
+        (r'[\w-]+', Name.Decorator),
+        (r'#\{', String.Interpol, 'interpolation'),
+        default('#pop'),
+    ],
+
+    'class': [
+        (r'[\w-]+', Name.Class),
+        (r'#\{', String.Interpol, 'interpolation'),
+        default('#pop'),
+    ],
+
+    'id': [
+        (r'[\w-]+', Name.Namespace),
+        (r'#\{', String.Interpol, 'interpolation'),
+        default('#pop'),
+    ],
+
+    'for': [
+        (r'(from|to|through)', Operator.Word),
+        include('value'),
+    ],
+}
+
+
+def _indentation(lexer, match, ctx):
+    indentation = match.group(0)
+    yield match.start(), Text, indentation
+    ctx.last_indentation = indentation
+    ctx.pos = match.end()
+
+    if hasattr(ctx, 'block_state') and ctx.block_state and \
+            indentation.startswith(ctx.block_indentation) and \
+            indentation != ctx.block_indentation:
+        ctx.stack.append(ctx.block_state)
+    else:
+        ctx.block_state = None
+        ctx.block_indentation = None
+        ctx.stack.append('content')
+
+
+def _starts_block(token, state):
+    def callback(lexer, match, ctx):
+        yield match.start(), token, match.group(0)
+
+        if hasattr(ctx, 'last_indentation'):
+            ctx.block_indentation = ctx.last_indentation
+        else:
+            ctx.block_indentation = ''
+
+        ctx.block_state = state
+        ctx.pos = match.end()
+
+    return callback
+
+
+class SassLexer(ExtendedRegexLexer):
+    """
+    For Sass stylesheets.
+
+    .. versionadded:: 1.3
+    """
+
+    name = 'Sass'
+    aliases = ['sass']
+    filenames = ['*.sass']
+    mimetypes = ['text/x-sass']
+
+    flags = re.IGNORECASE | re.MULTILINE
+
+    tokens = {
+        'root': [
+            (r'[ \t]*\n', Text),
+            (r'[ \t]*', _indentation),
+        ],
+
+        'content': [
+            (r'//[^\n]*', _starts_block(Comment.Single, 'single-comment'),
+             'root'),
+            (r'/\*[^\n]*', _starts_block(Comment.Multiline, 'multi-comment'),
+             'root'),
+            (r'@import', Keyword, 'import'),
+            (r'@for', Keyword, 'for'),
+            (r'@(debug|warn|if|while)', Keyword, 'value'),
+            (r'(@mixin)( [\w-]+)', bygroups(Keyword, Name.Function), 'value'),
+            (r'(@include)( [\w-]+)', bygroups(Keyword, Name.Decorator), 'value'),
+            (r'@extend', Keyword, 'selector'),
+            (r'@[\w-]+', Keyword, 'selector'),
+            (r'=[\w-]+', Name.Function, 'value'),
+            (r'\+[\w-]+', Name.Decorator, 'value'),
+            (r'([!$][\w-]\w*)([ \t]*(?:(?:\|\|)?=|:))',
+             bygroups(Name.Variable, Operator), 'value'),
+            (r':', Name.Attribute, 'old-style-attr'),
+            (r'(?=.+?[=:]([^a-z]|$))', Name.Attribute, 'new-style-attr'),
+            default('selector'),
+        ],
+
+        'single-comment': [
+            (r'.+', Comment.Single),
+            (r'\n', Text, 'root'),
+        ],
+
+        'multi-comment': [
+            (r'.+', Comment.Multiline),
+            (r'\n', Text, 'root'),
+        ],
+
+        'import': [
+            (r'[ \t]+', Text),
+            (r'\S+', String),
+            (r'\n', Text, 'root'),
+        ],
+
+        'old-style-attr': [
+            (r'[^\s:="\[]+', Name.Attribute),
+            (r'#\{', String.Interpol, 'interpolation'),
+            (r'[ \t]*=', Operator, 'value'),
+            default('value'),
+        ],
+
+        'new-style-attr': [
+            (r'[^\s:="\[]+', Name.Attribute),
+            (r'#\{', String.Interpol, 'interpolation'),
+            (r'[ \t]*[=:]', Operator, 'value'),
+        ],
+
+        'inline-comment': [
+            (r"(\\#|#(?=[^\n{])|\*(?=[^\n/])|[^\n#*])+", Comment.Multiline),
+            (r'#\{', String.Interpol, 'interpolation'),
+            (r"\*/", Comment, '#pop'),
+        ],
+    }
+    for group, common in iteritems(common_sass_tokens):
+        tokens[group] = copy.copy(common)
+    tokens['value'].append((r'\n', Text, 'root'))
+    tokens['selector'].append((r'\n', Text, 'root'))
+
+
+class ScssLexer(RegexLexer):
+    """
+    For SCSS stylesheets.
+    """
+
+    name = 'SCSS'
+    aliases = ['scss']
+    filenames = ['*.scss']
+    mimetypes = ['text/x-scss']
+
+    flags = re.IGNORECASE | re.DOTALL
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+            (r'//.*?\n', Comment.Single),
+            (r'/\*.*?\*/', Comment.Multiline),
+            (r'@import', Keyword, 'value'),
+            (r'@for', Keyword, 'for'),
+            (r'@(debug|warn|if|while)', Keyword, 'value'),
+            (r'(@mixin)( [\w-]+)', bygroups(Keyword, Name.Function), 'value'),
+            (r'(@include)( [\w-]+)', bygroups(Keyword, Name.Decorator), 'value'),
+            (r'@extend', Keyword, 'selector'),
+            (r'(@media)(\s+)', bygroups(Keyword, Text), 'value'),
+            (r'@[\w-]+', Keyword, 'selector'),
+            (r'(\$[\w-]*\w)([ \t]*:)', bygroups(Name.Variable, Operator), 'value'),
+            # TODO: broken, and prone to infinite loops.
+            # (r'(?=[^;{}][;}])', Name.Attribute, 'attr'),
+            # (r'(?=[^;{}:]+:[^a-z])', Name.Attribute, 'attr'),
+            default('selector'),
+        ],
+
+        'attr': [
+            (r'[^\s:="\[]+', Name.Attribute),
+            (r'#\{', String.Interpol, 'interpolation'),
+            (r'[ \t]*:', Operator, 'value'),
+            default('#pop'),
+        ],
+
+        'inline-comment': [
+            (r"(\\#|#(?=[^{])|\*(?=[^/])|[^#*])+", Comment.Multiline),
+            (r'#\{', String.Interpol, 'interpolation'),
+            (r"\*/", Comment, '#pop'),
+        ],
+    }
+    for group, common in iteritems(common_sass_tokens):
+        tokens[group] = copy.copy(common)
+    tokens['value'].extend([(r'\n', Text), (r'[;{}]', Punctuation, '#pop')])
+    tokens['selector'].extend([(r'\n', Text), (r'[;{}]', Punctuation, '#pop')])
+
+
+class LessCssLexer(CssLexer):
+    """
+    For `LESS <http://lesscss.org/>`_ styleshets.
+
+    .. versionadded:: 2.1
+    """
+
+    name = 'LessCss'
+    aliases = ['less']
+    filenames = ['*.less']
+    mimetypes = ['text/x-less-css']
+
+    tokens = {
+        'root': [
+            (r'@\w+', Name.Variable),
+            inherit,
+        ],
+        'content': [
+            (r'\{', Punctuation, '#push'),
+            inherit,
+        ],
+    }

packages/wakatime/packages/pygments/lexers/d.py

@@ -5,7 +5,7 @@
 
     Lexers for D languages.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/dalvik.py

@@ -5,7 +5,7 @@
 
     Pygments lexers for Dalvik VM-related languages.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """