v7.0.15

fix UnicodeDecodeError: 'ascii' codec can't decode byte 0xd0 in posiion 10: ordinal not in range(128) on plugin_loaded (ST2)

HISTORY.rst

@@ -3,6 +3,241 @@ History
 -------
 
 
+7.0.15 (2017-02-13)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.2.2.
+- Upgrade pygments library to v2.2.0 for improved language detection.
+
+
+7.0.14 (2017-02-08)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.2.1.
+- Allow boolean or list of regex patterns for hidefilenames config setting.
+
+
+7.0.13 (2016-11-11)
+++++++++++++++++++
+
+- Support old Sublime Text with Python 2.6.
+- Fix bug that prevented reading default api key from existing config file.
+
+
+7.0.12 (2016-10-24)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.2.0.
+- Exit with status code 104 when api key is missing or invalid. Exit with
+  status code 103 when config file missing or invalid.
+- New WAKATIME_HOME env variable for setting path to config and log files.
+- Improve debug warning message from unsupported dependency parsers.
+
+
+7.0.11 (2016-09-23)
+++++++++++++++++++
+
+- Handle UnicodeDecodeError when when logging. Related to #68.
+
+
+7.0.10 (2016-09-22)
+++++++++++++++++++
+
+- Handle UnicodeDecodeError when looking for python. Fixes #68.
+- Upgrade wakatime-cli to v6.0.9.
+
+
+7.0.9 (2016-09-02)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.0.8.
+
+
+7.0.8 (2016-07-21)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to master version to fix debug logging encoding bug.
+
+
+7.0.7 (2016-07-06)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.0.7.
+- Handle unknown exceptions from requests library by deleting cached session
+  object because it could be from a previous conflicting version.
+- New hostname setting in config file to set machine hostname. Hostname
+  argument takes priority over hostname from config file.
+- Prevent logging unrelated exception when logging tracebacks.
+- Use correct namespace for pygments.lexers.ClassNotFound exception so it is
+  caught when dependency detection not available for a language.
+
+
+7.0.6 (2016-06-13)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.0.5.
+- Upgrade pygments to v2.1.3 for better language coverage.
+
+
+7.0.5 (2016-06-08)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to master version to fix bug in urllib3 package causing
+  unhandled retry exceptions.
+- Prevent tracking git branch with detached head.
+
+
+7.0.4 (2016-05-21)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.0.3.
+- Upgrade requests dependency to v2.10.0.
+- Support for SOCKS proxies.
+
+
+7.0.3 (2016-05-16)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.0.2.
+- Prevent popup on Mac when xcode-tools is not installed.
+
+
+7.0.2 (2016-04-29)
+++++++++++++++++++
+
+- Prevent implicit unicode decoding from string format when logging output
+  from Python version check.
+
+
+7.0.1 (2016-04-28)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.0.1.
+- Fix bug which prevented plugin from being sent with extra heartbeats.
+
+
+7.0.0 (2016-04-28)
+++++++++++++++++++
+
+- Queue heartbeats and send to wakatime-cli after 4 seconds.
+- Nest settings menu under Package Settings.
+- Upgrade wakatime-cli to v6.0.0.
+- Increase default network timeout to 60 seconds when sending heartbeats to
+  the api.
+- New --extra-heartbeats command line argument for sending a JSON array of
+  extra queued heartbeats to STDIN.
+- Change --entitytype command line argument to --entity-type.
+- No longer allowing --entity-type of url.
+- Support passing an alternate language to cli to be used when a language can
+  not be guessed from the code file.
+
+
+6.0.8 (2016-04-18)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v5.0.0.
+- Support regex patterns in projectmap config section for renaming projects.
+- Upgrade pytz to v2016.3.
+- Upgrade tzlocal to v1.2.2.
+
+
+6.0.7 (2016-03-11)
+++++++++++++++++++
+
+- Fix bug causing RuntimeError when finding Python location
+
+
+6.0.6 (2016-03-06)
+++++++++++++++++++
+
+- upgrade wakatime-cli to v4.1.13
+- encode TimeZone as utf-8 before adding to headers
+- encode X-Machine-Name as utf-8 before adding to headers
+
+
+6.0.5 (2016-03-06)
+++++++++++++++++++
+
+- upgrade wakatime-cli to v4.1.11
+- encode machine hostname as Unicode when adding to X-Machine-Name header
+
+
+6.0.4 (2016-01-15)
+++++++++++++++++++
+
+- fix UnicodeDecodeError on ST2 with non-English locale
+
+
+6.0.3 (2016-01-11)
+++++++++++++++++++
+
+- upgrade wakatime-cli core to v4.1.10
+- accept 201 or 202 response codes as success from api
+- upgrade requests package to v2.9.1
+
+
+6.0.2 (2016-01-06)
+++++++++++++++++++
+
+- upgrade wakatime-cli core to v4.1.9
+- improve C# dependency detection
+- correctly log exception tracebacks
+- log all unknown exceptions to wakatime.log file
+- disable urllib3 SSL warning from every request
+- detect dependencies from golang files
+- use api.wakatime.com for sending heartbeats
+
+
+6.0.1 (2016-01-01)
+++++++++++++++++++
+
+- use embedded python if system python is broken, or doesn't output a version number
+- log output from wakatime-cli in ST console when in debug mode
+
+
+6.0.0 (2015-12-01)
+++++++++++++++++++
+
+- use embeddable Python instead of installing on Windows
+
+
+5.0.1 (2015-10-06)
+++++++++++++++++++
+
+- look for python in system PATH again
+
+
+5.0.0 (2015-10-02)
+++++++++++++++++++
+
+- improve logging with levels and log function
+- switch registry warnings to debug log level
+
+
+4.0.20 (2015-10-01)
+++++++++++++++++++
+
+- correctly find python binary in non-Windows environments
+
+
+4.0.19 (2015-10-01)
+++++++++++++++++++
+
+- handle case where ST builtin python does not have _winreg or winreg module
+
+
+4.0.18 (2015-10-01)
+++++++++++++++++++
+
+- find python location from windows registry
+
+
+4.0.17 (2015-10-01)
+++++++++++++++++++
+
+- download python in non blocking background thread for Windows machines
+
+
 4.0.16 (2015-09-29)
 ++++++++++++++++++
 

Main.sublime-menu

@@ -6,24 +6,37 @@
         "children":
         [
             {
-                "caption": "WakaTime",
-                "mnemonic": "W",
-                "id": "wakatime-settings",
+                "caption": "Package Settings",
+                "mnemonic": "P",
+                "id": "package-settings",
                 "children":
                 [
                     {
-                        "command": "open_file", "args":
-                        {
-                            "file": "${packages}/WakaTime/WakaTime.sublime-settings"
-                        },
-                        "caption": "Settings – Default"
-                    },
-                    {
-                        "command": "open_file", "args":
-                        {
-                            "file": "${packages}/User/WakaTime.sublime-settings"
-                        },
-                        "caption": "Settings – User"
+                        "caption": "WakaTime",
+                        "mnemonic": "W",
+                        "id": "wakatime-settings",
+                        "children":
+                        [
+                            {
+                                "command": "open_file", "args":
+                                {
+                                    "file": "${packages}/WakaTime/WakaTime.sublime-settings"
+                                },
+                                "caption": "Settings – Default"
+                            },
+                            {
+                                "command": "open_file", "args":
+                                {
+                                    "file": "${packages}/User/WakaTime.sublime-settings"
+                                },
+                                "caption": "Settings – User"
+                            },
+                            {
+                                "command": "wakatime_dashboard",
+                                "args": {},
+                                "caption": "WakaTime Dashboard"
+                            }
+                        ]
                     }
                 ]
             }

README.md

@@ -1,13 +1,12 @@
 sublime-wakatime
 ================
 
-Sublime Text 2 & 3 plugin to quantify your coding using https://wakatime.com/.
+Metrics, insights, and time tracking automatically generated from your programming activity.
+
 
 Installation
 ------------
 
-Heads Up! For Sublime Text 2 on Windows & Linux, WakaTime depends on [Python](http://www.python.org/getit/) being installed to work correctly.
-    
 1. Install [Package Control](https://packagecontrol.io/installation).
 
 2. Using [Package Control](https://packagecontrol.io/docs/usage):
@@ -24,17 +23,31 @@ Heads Up! For Sublime Text 2 on Windows & Linux, WakaTime depends on [Python](ht
 
 5. Visit https://wakatime.com/dashboard to see your logged time.
 
+
 Screen Shots
 ------------
 
-![Project Overview](https://wakatime.com/static/img/ScreenShots/ScreenShot-2014-10-29.png)
+![Project Overview](https://wakatime.com/static/img/ScreenShots/Screen-Shot-2016-03-21.png)
+
+
+Unresponsive Plugin Warning
+---------------------------
+
+In Sublime Text 2, if you get a warning message:
+
+    A plugin (WakaTime) may be making Sublime Text unresponsive by taking too long (0.017332s) in its on_modified callback.
+
+To fix this, go to `Preferences > Settings - User` then add the following setting:
+
+`"detect_slow_plugins": false`
+
 
 Troubleshooting
 ---------------
 
 First, turn on debug mode in your `WakaTime.sublime-settings` file.
 
-![sublime user settings](https://wakatime.com/static/img/ScreenShots/sublime-wakatime-settings-menu.png)
+![sublime user settings](https://wakatime.com/static/img/ScreenShots/sublime-wakatime-settings-menu.png?v=3)
 
 Add the line: `"debug": true`
 

WakaTime.py

@@ -7,22 +7,88 @@ Website:     https://wakatime.com/
 ==========================================================="""
 
 
-__version__ = '4.0.16'
+__version__ = '7.0.15'
 
 
 import sublime
 import sublime_plugin
 
-import glob
+import contextlib
+import json
 import os
 import platform
+import re
 import sys
 import time
 import threading
+import traceback
 import urllib
 import webbrowser
 from datetime import datetime
-from subprocess import Popen
+from subprocess import Popen, STDOUT, PIPE
+from zipfile import ZipFile
+try:
+    import _winreg as winreg  # py2
+except ImportError:
+    try:
+        import winreg  # py3
+    except ImportError:
+        winreg = None
+try:
+    import Queue as queue  # py2
+except ImportError:
+    import queue  # py3
+
+
+is_py2 = (sys.version_info[0] == 2)
+is_py3 = (sys.version_info[0] == 3)
+
+if is_py2:
+    def u(text):
+        if text is None:
+            return None
+        if isinstance(text, unicode):
+            return text
+        try:
+            return text.decode('utf-8')
+        except:
+            try:
+                return text.decode(sys.getdefaultencoding())
+            except:
+                try:
+                    return unicode(text)
+                except:
+                    try:
+                        return text.decode('utf-8', 'replace')
+                    except:
+                        try:
+                            return unicode(str(text))
+                        except:
+                            return unicode('')
+
+elif is_py3:
+    def u(text):
+        if text is None:
+            return None
+        if isinstance(text, bytes):
+            try:
+                return text.decode('utf-8')
+            except:
+                try:
+                    return text.decode(sys.getdefaultencoding())
+                except:
+                    pass
+        try:
+            return str(text)
+        except:
+            return text.decode('utf-8', 'replace')
+
+else:
+    raise Exception('Unsupported Python version: {0}.{1}.{2}'.format(
+        sys.version_info[0],
+        sys.version_info[1],
+        sys.version_info[2],
+    ))
 
 
 # globals
@@ -37,19 +103,81 @@ LAST_HEARTBEAT = {
     'file': None,
     'is_write': False,
 }
-LOCK = threading.RLock()
 PYTHON_LOCATION = None
+HEARTBEATS = queue.Queue()
+
+
+# Log Levels
+DEBUG = 'DEBUG'
+INFO = 'INFO'
+WARNING = 'WARNING'
+ERROR = 'ERROR'
 
 
 # add wakatime package to path
 sys.path.insert(0, os.path.join(PLUGIN_DIR, 'packages'))
 try:
-    from wakatime.base import parseConfigFile
+    from wakatime.main import parseConfigFile
 except ImportError:
     pass
 
 
-def createConfigFile():
+def set_timeout(callback, seconds):
+    """Runs the callback after the given seconds delay.
+
+    If this is Sublime Text 3, runs the callback on an alternate thread. If this
+    is Sublime Text 2, runs the callback in the main thread.
+    """
+
+    milliseconds = int(seconds * 1000)
+    try:
+        sublime.set_timeout_async(callback, milliseconds)
+    except AttributeError:
+        sublime.set_timeout(callback, milliseconds)
+
+
+def log(lvl, message, *args, **kwargs):
+    try:
+        if lvl == DEBUG and not SETTINGS.get('debug'):
+            return
+        msg = message
+        if len(args) > 0:
+            msg = message.format(*args)
+        elif len(kwargs) > 0:
+            msg = message.format(**kwargs)
+        try:
+            print('[WakaTime] [{lvl}] {msg}'.format(lvl=lvl, msg=msg))
+        except UnicodeDecodeError:
+            print(u('[WakaTime] [{lvl}] {msg}').format(lvl=lvl, msg=u(msg)))
+    except RuntimeError:
+        set_timeout(lambda: log(lvl, message, *args, **kwargs), 0)
+
+
+def resources_folder():
+    if platform.system() == 'Windows':
+        return os.path.join(os.getenv('APPDATA'), 'WakaTime')
+    else:
+        return os.path.join(os.path.expanduser('~'), '.wakatime')
+
+
+def update_status_bar(status):
+    """Updates the status bar."""
+
+    try:
+        if SETTINGS.get('status_bar_message'):
+            msg = datetime.now().strftime(SETTINGS.get('status_bar_message_fmt'))
+            if '{status}' in msg:
+                msg = msg.format(status=status)
+
+            active_window = sublime.active_window()
+            if active_window:
+                for view in active_window.views():
+                    view.set_status('wakatime', msg)
+    except RuntimeError:
+        set_timeout(lambda: update_status_bar(status), 0)
+
+
+def create_config_file():
     """Creates the .wakatime.cfg INI file in $HOME directory, if it does
     not already exist.
     """
@@ -70,7 +198,7 @@ def createConfigFile():
 def prompt_api_key():
     global SETTINGS
 
-    createConfigFile()
+    create_config_file()
 
     default_key = ''
     try:
@@ -93,35 +221,135 @@ def prompt_api_key():
             window.show_input_panel('[WakaTime] Enter your wakatime.com api key:', default_key, got_key, None, None)
             return True
         else:
-            print('[WakaTime] Error: Could not prompt for api key because no window found.')
+            log(ERROR, 'Could not prompt for api key because no window found.')
     return False
 
 
 def python_binary():
-    global PYTHON_LOCATION
     if PYTHON_LOCATION is not None:
         return PYTHON_LOCATION
+
+    # look for python in PATH and common install locations
     paths = [
-        "pythonw",
-        "python",
-        "/usr/local/bin/python",
-        "/usr/bin/python",
+        os.path.join(resources_folder(), 'python'),
+        None,
+        '/',
+        '/usr/local/bin/',
+        '/usr/bin/',
     ]
     for path in paths:
-        try:
-            Popen([path, '--version'])
-            PYTHON_LOCATION = path
+        path = find_python_in_folder(path)
+        if path is not None:
+            set_python_binary_location(path)
             return path
-        except:
-            pass
-    for path in glob.iglob('/python*'):
-        path = os.path.realpath(os.path.join(path, 'pythonw'))
-        try:
-            Popen([path, '--version'])
-            PYTHON_LOCATION = path
+
+    # look for python in windows registry
+    path = find_python_from_registry(r'SOFTWARE\Python\PythonCore')
+    if path is not None:
+        set_python_binary_location(path)
+        return path
+    path = find_python_from_registry(r'SOFTWARE\Wow6432Node\Python\PythonCore')
+    if path is not None:
+        set_python_binary_location(path)
+        return path
+
+    return None
+
+
+def set_python_binary_location(path):
+    global PYTHON_LOCATION
+    PYTHON_LOCATION = path
+    log(DEBUG, 'Found Python at: {0}'.format(path))
+
+
+def find_python_from_registry(location, reg=None):
+    if platform.system() != 'Windows' or winreg is None:
+        return None
+
+    if reg is None:
+        path = find_python_from_registry(location, reg=winreg.HKEY_CURRENT_USER)
+        if path is None:
+            path = find_python_from_registry(location, reg=winreg.HKEY_LOCAL_MACHINE)
+        return path
+
+    val = None
+    sub_key = 'InstallPath'
+    compiled = re.compile(r'^\d+\.\d+$')
+
+    try:
+        with winreg.OpenKey(reg, location) as handle:
+            versions = []
+            try:
+                for index in range(1024):
+                    version = winreg.EnumKey(handle, index)
+                    try:
+                        if compiled.search(version):
+                            versions.append(version)
+                    except re.error:
+                        pass
+            except EnvironmentError:
+                pass
+            versions.sort(reverse=True)
+            for version in versions:
+                try:
+                    path = winreg.QueryValue(handle, version + '\\' + sub_key)
+                    if path is not None:
+                        path = find_python_in_folder(path)
+                        if path is not None:
+                            log(DEBUG, 'Found python from {reg}\\{key}\\{version}\\{sub_key}.'.format(
+                                reg=reg,
+                                key=location,
+                                version=version,
+                                sub_key=sub_key,
+                            ))
+                            return path
+                except WindowsError:
+                    log(DEBUG, 'Could not read registry value "{reg}\\{key}\\{version}\\{sub_key}".'.format(
+                        reg=reg,
+                        key=location,
+                        version=version,
+                        sub_key=sub_key,
+                    ))
+    except WindowsError:
+        log(DEBUG, 'Could not read registry value "{reg}\\{key}".'.format(
+            reg=reg,
+            key=location,
+        ))
+    except:
+        log(ERROR, 'Could not read registry value "{reg}\\{key}":\n{exc}'.format(
+            reg=reg,
+            key=location,
+            exc=traceback.format_exc(),
+        ))
+
+    return val
+
+
+def find_python_in_folder(folder, headless=True):
+    pattern = re.compile(r'\d+\.\d+')
+
+    path = 'python'
+    if folder is not None:
+        path = os.path.realpath(os.path.join(folder, 'python'))
+    if headless:
+        path = u(path) + u('w')
+    log(DEBUG, u('Looking for Python at: {0}').format(u(path)))
+    try:
+        process = Popen([path, '--version'], stdout=PIPE, stderr=STDOUT)
+        output, err = process.communicate()
+        output = u(output).strip()
+        retcode = process.poll()
+        log(DEBUG, u('Python Version Output: {0}').format(output))
+        if not retcode and pattern.search(output):
             return path
-        except:
-            pass
+    except:
+        log(DEBUG, u(sys.exc_info()[1]))
+
+    if headless:
+        path = find_python_in_folder(folder, headless=False)
+        if path is not None:
+            return path
+
     return None
 
 
@@ -133,14 +361,14 @@ def obfuscate_apikey(command_list):
             apikey_index = num + 1
             break
     if apikey_index is not None and apikey_index < len(cmd):
-        cmd[apikey_index] = '********-****-****-****-********' + cmd[apikey_index][-4:]
+        cmd[apikey_index] = 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXX' + cmd[apikey_index][-4:]
     return cmd
 
 
-def enough_time_passed(now, last_heartbeat, is_write):
-    if now - last_heartbeat['time'] > HEARTBEAT_FREQUENCY * 60:
+def enough_time_passed(now, is_write):
+    if now - LAST_HEARTBEAT['time'] > HEARTBEAT_FREQUENCY * 60:
         return True
-    if is_write and now - last_heartbeat['time'] > 2:
+    if is_write and now - LAST_HEARTBEAT['time'] > 2:
         return True
     return False
 
@@ -184,135 +412,235 @@ def is_view_active(view):
     return False
 
 
-def handle_heartbeat(view, is_write=False):
+def handle_activity(view, is_write=False):
     window = view.window()
     if window is not None:
-        target_file = view.file_name()
-        project = window.project_data() if hasattr(window, 'project_data') else None
-        folders = window.folders()
-        thread = SendHeartbeatThread(target_file, view, is_write=is_write, project=project, folders=folders)
-        thread.start()
+        entity = view.file_name()
+        if entity:
+            timestamp = time.time()
+            last_file = LAST_HEARTBEAT['file']
+            if entity != last_file or enough_time_passed(timestamp, is_write):
+                project = window.project_data() if hasattr(window, 'project_data') else None
+                folders = window.folders()
+                append_heartbeat(entity, timestamp, is_write, view, project, folders)
 
 
-class SendHeartbeatThread(threading.Thread):
+def append_heartbeat(entity, timestamp, is_write, view, project, folders):
+    global LAST_HEARTBEAT
 
-    def __init__(self, target_file, view, is_write=False, project=None, folders=None, force=False):
+    # add this heartbeat to queue
+    heartbeat = {
+        'entity': entity,
+        'timestamp': timestamp,
+        'is_write': is_write,
+        'cursorpos': view.sel()[0].begin() if view.sel() else None,
+        'project': project,
+        'folders': folders,
+    }
+    HEARTBEATS.put_nowait(heartbeat)
+
+    # make this heartbeat the LAST_HEARTBEAT
+    LAST_HEARTBEAT = {
+        'file': entity,
+        'time': timestamp,
+        'is_write': is_write,
+    }
+
+    # process the queue of heartbeats in the future
+    seconds = 4
+    set_timeout(process_queue, seconds)
+
+
+def process_queue():
+    try:
+        heartbeat = HEARTBEATS.get_nowait()
+    except queue.Empty:
+        return
+
+    has_extra_heartbeats = False
+    extra_heartbeats = []
+    try:
+        while True:
+            extra_heartbeats.append(HEARTBEATS.get_nowait())
+            has_extra_heartbeats = True
+    except queue.Empty:
+        pass
+
+    thread = SendHeartbeatsThread(heartbeat)
+    if has_extra_heartbeats:
+        thread.add_extra_heartbeats(extra_heartbeats)
+    thread.start()
+
+
+class SendHeartbeatsThread(threading.Thread):
+    """Non-blocking thread for sending heartbeats to api.
+    """
+
+    def __init__(self, heartbeat):
         threading.Thread.__init__(self)
-        self.lock = LOCK
-        self.target_file = target_file
-        self.is_write = is_write
-        self.project = project
-        self.folders = folders
-        self.force = force
+
         self.debug = SETTINGS.get('debug')
         self.api_key = SETTINGS.get('api_key', '')
         self.ignore = SETTINGS.get('ignore', [])
-        self.last_heartbeat = LAST_HEARTBEAT.copy()
-        self.cursorpos = view.sel()[0].begin() if view.sel() else None
-        self.view = view
+
+        self.heartbeat = heartbeat
+        self.has_extra_heartbeats = False
+
+    def add_extra_heartbeats(self, extra_heartbeats):
+        self.has_extra_heartbeats = True
+        self.extra_heartbeats = extra_heartbeats
 
     def run(self):
-        with self.lock:
-            if self.target_file:
-                self.timestamp = time.time()
-                if self.force or self.target_file != self.last_heartbeat['file'] or enough_time_passed(self.timestamp, self.last_heartbeat, self.is_write):
-                    self.send_heartbeat()
+        """Running in background thread."""
 
-    def send_heartbeat(self):
-        if not self.api_key:
-            print('[WakaTime] Error: missing api key.')
-            return
-        ua = 'sublime/%d sublime-wakatime/%s' % (ST_VERSION, __version__)
-        cmd = [
-            API_CLIENT,
-            '--file', self.target_file,
-            '--time', str('%f' % self.timestamp),
-            '--plugin', ua,
-            '--key', str(bytes.decode(self.api_key.encode('utf8'))),
-        ]
-        if self.is_write:
-            cmd.append('--write')
-        if self.project and self.project.get('name'):
-            cmd.extend(['--alternate-project', self.project.get('name')])
-        elif self.folders:
-            project_name = find_project_from_folders(self.folders, self.target_file)
+        self.send_heartbeats()
+
+    def build_heartbeat(self, entity=None, timestamp=None, is_write=None,
+                        cursorpos=None, project=None, folders=None):
+        """Returns a dict for passing to wakatime-cli as arguments."""
+
+        heartbeat = {
+            'entity': entity,
+            'timestamp': timestamp,
+            'is_write': is_write,
+        }
+
+        if project and project.get('name'):
+            heartbeat['alternate_project'] = project.get('name')
+        elif folders:
+            project_name = find_project_from_folders(folders, entity)
             if project_name:
-                cmd.extend(['--alternate-project', project_name])
-        if self.cursorpos is not None:
-            cmd.extend(['--cursorpos', '{0}'.format(self.cursorpos)])
-        for pattern in self.ignore:
-            cmd.extend(['--ignore', pattern])
-        if self.debug:
-            cmd.append('--verbose')
+                heartbeat['alternate_project'] = project_name
+
+        if cursorpos is not None:
+            heartbeat['cursorpos'] = '{0}'.format(cursorpos)
+
+        return heartbeat
+
+    def send_heartbeats(self):
         if python_binary():
-            cmd.insert(0, python_binary())
+            heartbeat = self.build_heartbeat(**self.heartbeat)
+            ua = 'sublime/%d sublime-wakatime/%s' % (ST_VERSION, __version__)
+            cmd = [
+                python_binary(),
+                API_CLIENT,
+                '--entity', heartbeat['entity'],
+                '--time', str('%f' % heartbeat['timestamp']),
+                '--plugin', ua,
+            ]
+            if self.api_key:
+                cmd.extend(['--key', str(bytes.decode(self.api_key.encode('utf8')))])
+            if heartbeat['is_write']:
+                cmd.append('--write')
+            if heartbeat.get('alternate_project'):
+                cmd.extend(['--alternate-project', heartbeat['alternate_project']])
+            if heartbeat.get('cursorpos') is not None:
+                cmd.extend(['--cursorpos', heartbeat['cursorpos']])
+            for pattern in self.ignore:
+                cmd.extend(['--ignore', pattern])
             if self.debug:
-                print('[WakaTime] %s' % ' '.join(obfuscate_apikey(cmd)))
-            if platform.system() == 'Windows':
-                Popen(cmd, shell=False)
+                cmd.append('--verbose')
+            if self.has_extra_heartbeats:
+                cmd.append('--extra-heartbeats')
+                stdin = PIPE
+                extra_heartbeats = [self.build_heartbeat(**x) for x in self.extra_heartbeats]
+                extra_heartbeats = json.dumps(extra_heartbeats)
             else:
-                with open(os.path.join(os.path.expanduser('~'), '.wakatime.log'), 'a') as stderr:
-                    Popen(cmd, stderr=stderr)
-            self.sent()
+                extra_heartbeats = None
+                stdin = None
+
+            log(DEBUG, ' '.join(obfuscate_apikey(cmd)))
+            try:
+                process = Popen(cmd, stdin=stdin, stdout=PIPE, stderr=STDOUT)
+                inp = None
+                if self.has_extra_heartbeats:
+                    inp = "{0}\n".format(extra_heartbeats)
+                    inp = inp.encode('utf-8')
+                output, err = process.communicate(input=inp)
+                output = u(output)
+                retcode = process.poll()
+                if (not retcode or retcode == 102) and not output:
+                    self.sent()
+                else:
+                    update_status_bar('Error')
+                if retcode:
+                    log(DEBUG if retcode == 102 else ERROR, 'wakatime-core exited with status: {0}'.format(retcode))
+                if output:
+                    log(ERROR, u('wakatime-core output: {0}').format(output))
+            except:
+                log(ERROR, u(sys.exc_info()[1]))
+                update_status_bar('Error')
+
         else:
-            print('[WakaTime] Error: Unable to find python binary.')
+            log(ERROR, 'Unable to find python binary.')
+            update_status_bar('Error')
 
     def sent(self):
-        sublime.set_timeout(self.set_status_bar, 0)
-        sublime.set_timeout(self.set_last_heartbeat, 0)
+        update_status_bar('OK')
 
-    def set_status_bar(self):
-        if SETTINGS.get('status_bar_message'):
-            self.view.set_status('wakatime', datetime.now().strftime(SETTINGS.get('status_bar_message_fmt')))
 
-    def set_last_heartbeat(self):
-        global LAST_HEARTBEAT
-        LAST_HEARTBEAT = {
-            'file': self.target_file,
-            'time': self.timestamp,
-            'is_write': self.is_write,
-        }
+def download_python():
+    thread = DownloadPython()
+    thread.start()
+
+
+class DownloadPython(threading.Thread):
+    """Non-blocking thread for extracting embeddable Python on Windows machines.
+    """
+
+    def run(self):
+        log(INFO, 'Downloading embeddable Python...')
+
+        ver = '3.5.2'
+        arch = 'amd64' if platform.architecture()[0] == '64bit' else 'win32'
+        url = 'https://www.python.org/ftp/python/{ver}/python-{ver}-embed-{arch}.zip'.format(
+            ver=ver,
+            arch=arch,
+        )
+
+        if not os.path.exists(resources_folder()):
+            os.makedirs(resources_folder())
+
+        zip_file = os.path.join(resources_folder(), 'python.zip')
+        try:
+            urllib.urlretrieve(url, zip_file)
+        except AttributeError:
+            urllib.request.urlretrieve(url, zip_file)
+
+        log(INFO, 'Extracting Python...')
+        with contextlib.closing(ZipFile(zip_file)) as zf:
+            path = os.path.join(resources_folder(), 'python')
+            zf.extractall(path)
+
+        try:
+            os.remove(zip_file)
+        except:
+            pass
+
+        log(INFO, 'Finished extracting Python.')
 
 
 def plugin_loaded():
     global SETTINGS
-    print('[WakaTime] Initializing WakaTime plugin v%s' % __version__)
+    SETTINGS = sublime.load_settings(SETTINGS_FILE)
+
+    log(INFO, 'Initializing WakaTime plugin v%s' % __version__)
+    update_status_bar('Initializing')
 
     if not python_binary():
-        print('[WakaTime] Warning: Python binary not found.')
+        log(WARNING, 'Python binary not found.')
         if platform.system() == 'Windows':
-            install_python()
+            set_timeout(download_python, 0)
         else:
             sublime.error_message("Unable to find Python binary!\nWakaTime needs Python to work correctly.\n\nGo to https://www.python.org/downloads")
             return
 
-    SETTINGS = sublime.load_settings(SETTINGS_FILE)
     after_loaded()
 
 
 def after_loaded():
     if not prompt_api_key():
-        sublime.set_timeout(after_loaded, 500)
-
-
-def install_python():
-    print('[WakaTime] Downloading and installing python...')
-    url = 'https://www.python.org/ftp/python/3.4.3/python-3.4.3.msi'
-    if platform.architecture()[0] == '64bit':
-        url = 'https://www.python.org/ftp/python/3.4.3/python-3.4.3.amd64.msi'
-    python_msi = os.path.join(os.path.expanduser('~'), 'python.msi')
-    try:
-        urllib.urlretrieve(url, python_msi)
-    except AttributeError:
-        urllib.request.urlretrieve(url, python_msi)
-    args = [
-        'msiexec',
-        '/i',
-        python_msi,
-        '/norestart',
-        '/qb!',
-    ]
-    Popen(args)
+        set_timeout(after_loaded, 0.5)
 
 
 # need to call plugin_loaded because only ST3 will auto-call it
@@ -323,15 +651,15 @@ if ST_VERSION < 3000:
 class WakatimeListener(sublime_plugin.EventListener):
 
     def on_post_save(self, view):
-        handle_heartbeat(view, is_write=True)
+        handle_activity(view, is_write=True)
 
     def on_selection_modified(self, view):
         if is_view_active(view):
-            handle_heartbeat(view)
+            handle_activity(view)
 
     def on_modified(self, view):
         if is_view_active(view):
-            handle_heartbeat(view)
+            handle_activity(view)
 
 
 class WakatimeDashboardCommand(sublime_plugin.ApplicationCommand):

WakaTime.sublime-settings

@@ -19,5 +19,5 @@
     "status_bar_message": true,
     
     // Status bar message format.
-    "status_bar_message_fmt": "WakaTime active %I:%M %p"
+    "status_bar_message_fmt": "WakaTime {status} %I:%M %p"
 }

packages/wakatime/__about__.py

@@ -1,9 +1,9 @@
 __title__ = 'wakatime'
 __description__ = 'Common interface to the WakaTime api.'
 __url__ = 'https://github.com/wakatime/wakatime'
-__version_info__ = ('4', '1', '8')
+__version_info__ = ('6', '2', '2')
 __version__ = '.'.join(__version_info__)
 __author__ = 'Alan Hamlett'
 __author_email__ = 'alan@wakatime.com'
 __license__ = 'BSD'
-__copyright__ = 'Copyright 2014 Alan Hamlett'
+__copyright__ = 'Copyright 2016 Alan Hamlett'

packages/wakatime/compat.py

@@ -31,7 +31,7 @@ if is_py2:  # pragma: nocover
                 try:
                     return unicode(text)
                 except:
-                    return text
+                    return text.decode('utf-8', 'replace')
     open = codecs.open
     basestring = basestring
 
@@ -52,7 +52,7 @@ elif is_py3:  # pragma: nocover
         try:
             return str(text)
         except:
-            return text
+            return text.decode('utf-8', 'replace')
     open = open
     basestring = (str, bytes)
 

packages/wakatime/constants.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.constants
+    ~~~~~~~~~~~~~~~~~~
+
+    Constant variable definitions.
+
+    :copyright: (c) 2016 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+""" Success
+Exit code used when a heartbeat was sent successfully.
+"""
+SUCCESS = 0
+
+""" Api Error
+Exit code used when the WakaTime API returned an error.
+"""
+API_ERROR = 102
+
+""" Config File Parse Error
+Exit code used when the ~/.wakatime.cfg config file could not be parsed.
+"""
+CONFIG_FILE_PARSE_ERROR = 103
+
+""" Auth Error
+Exit code used when our api key is invalid.
+"""
+AUTH_ERROR = 104
+
+""" Unknown Error
+Exit code used when there was an unhandled exception.
+"""
+UNKNOWN_ERROR = 105
+
+""" Malformed Heartbeat Error
+Exit code used when the JSON input from `--extra-heartbeats` is malformed.
+"""
+MALFORMED_HEARTBEAT_ERROR = 106

packages/wakatime/dependencies/__init__.py

@@ -12,7 +12,6 @@
 import logging
 import re
 import sys
-import traceback
 
 from ..compat import u, open, import_module
 from ..exceptions import NotYetImplemented
@@ -68,7 +67,7 @@ class TokenParser(object):
                 pass
             try:
                 with open(self.source_file, 'r', encoding=sys.getfilesystemencoding()) as fh:
-                    return self.lexer.get_tokens_unprocessed(fh.read(512000))
+                    return self.lexer.get_tokens_unprocessed(fh.read(512000))  # pragma: nocover
             except:
                 pass
         return []
@@ -118,9 +117,9 @@ class DependencyParser(object):
             try:
                 self.parser = getattr(module, class_name)
             except AttributeError:
-                log.debug('Module {0} is missing class {1}'.format(module.__name__, class_name))
+                log.debug('Parsing dependencies not supported for {0}.{1}'.format(module_name, class_name))
         except ImportError:
-            log.debug(traceback.format_exc())
+            log.debug('Parsing dependencies not supported for {0}.{1}'.format(module_name, class_name))
 
     def parse(self):
         if self.parser:

packages/wakatime/dependencies/c_cpp.py

@@ -12,34 +12,6 @@
 from . import TokenParser
 
 
-class CppParser(TokenParser):
-    exclude = [
-        r'^stdio\.h$',
-        r'^stdlib\.h$',
-        r'^string\.h$',
-        r'^time\.h$',
-    ]
-
-    def parse(self):
-        for index, token, content in self.tokens:
-            self._process_token(token, content)
-        return self.dependencies
-
-    def _process_token(self, token, content):
-        if self.partial(token) == 'Preproc':
-            self._process_preproc(token, content)
-        else:
-            self._process_other(token, content)
-
-    def _process_preproc(self, token, content):
-        if content.strip().startswith('include ') or content.strip().startswith("include\t"):
-            content = content.replace('include', '', 1).strip().strip('"').strip('<').strip('>').strip()
-            self.append(content)
-
-    def _process_other(self, token, content):
-        pass
-
-
 class CParser(TokenParser):
     exclude = [
         r'^stdio\.h$',
@@ -47,6 +19,7 @@ class CParser(TokenParser):
         r'^string\.h$',
         r'^time\.h$',
     ]
+    state = None
 
     def parse(self):
         for index, token, content in self.tokens:
@@ -54,15 +27,25 @@ class CParser(TokenParser):
         return self.dependencies
 
     def _process_token(self, token, content):
-        if self.partial(token) == 'Preproc':
+        if self.partial(token) == 'Preproc' or self.partial(token) == 'PreprocFile':
             self._process_preproc(token, content)
         else:
             self._process_other(token, content)
 
     def _process_preproc(self, token, content):
-        if content.strip().startswith('include ') or content.strip().startswith("include\t"):
-            content = content.replace('include', '', 1).strip().strip('"').strip('<').strip('>').strip()
-            self.append(content)
+        if self.state == 'include':
+            if content != '\n' and content != '#':
+                content = content.strip().strip('"').strip('<').strip('>').strip()
+                self.append(content, truncate=True, separator='/')
+            self.state = None
+        elif content.strip().startswith('include'):
+            self.state = 'include'
+        else:
+            self.state = None
 
     def _process_other(self, token, content):
         pass
+
+
+class CppParser(CParser):
+    pass

packages/wakatime/dependencies/dotnet.py

@@ -10,9 +10,16 @@
 """
 
 from . import TokenParser
+from ..compat import u
 
 
 class CSharpParser(TokenParser):
+    exclude = [
+        r'^system$',
+        r'^microsoft$',
+    ]
+    state = None
+    buffer = u('')
 
     def parse(self):
         for index, token, content in self.tokens:
@@ -20,14 +27,38 @@ class CSharpParser(TokenParser):
         return self.dependencies
 
     def _process_token(self, token, content):
-        if self.partial(token) == 'Namespace':
+        if self.partial(token) == 'Keyword':
+            self._process_keyword(token, content)
+        if self.partial(token) == 'Namespace' or self.partial(token) == 'Name':
             self._process_namespace(token, content)
+        elif self.partial(token) == 'Punctuation':
+            self._process_punctuation(token, content)
         else:
             self._process_other(token, content)
 
+    def _process_keyword(self, token, content):
+        if content == 'using':
+            self.state = 'import'
+            self.buffer = u('')
+
     def _process_namespace(self, token, content):
-        if content != 'import' and content != 'package' and content != 'namespace':
-            self.append(content, truncate=True)
+        if self.state == 'import':
+            if u(content) != u('import') and u(content) != u('package') and u(content) != u('namespace') and u(content) != u('static'):
+                if u(content) == u(';'):  # pragma: nocover
+                    self._process_punctuation(token, content)
+                else:
+                    self.buffer += u(content)
+
+    def _process_punctuation(self, token, content):
+        if self.state == 'import':
+            if u(content) == u(';'):
+                self.append(self.buffer, truncate=True)
+                self.buffer = u('')
+                self.state = None
+            elif u(content) == u('='):
+                self.buffer = u('')
+            else:
+                self.buffer += u(content)
 
     def _process_other(self, token, content):
         pass

packages/wakatime/dependencies/go.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.languages.go
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Go code.
+
+    :copyright: (c) 2016 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+
+
+class GoParser(TokenParser):
+    state = None
+    parens = 0
+    aliases = 0
+    exclude = [
+        r'^"fmt"$',
+    ]
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        elif self.partial(token) == 'Punctuation':
+            self._process_punctuation(token, content)
+        elif self.partial(token) == 'String':
+            self._process_string(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        elif self.partial(token) == 'Other':
+            self._process_other(token, content)
+        else:
+            self._process_misc(token, content)
+
+    def _process_namespace(self, token, content):
+        self.state = content
+        self.parens = 0
+        self.aliases = 0
+
+    def _process_string(self, token, content):
+        if self.state == 'import':
+            self.append(content, truncate=False)
+
+    def _process_punctuation(self, token, content):
+        if content == '(':
+            self.parens += 1
+        elif content == ')':
+            self.parens -= 1
+        elif content == '.':
+            self.aliases += 1
+        else:
+            self.state = None
+
+    def _process_text(self, token, content):
+        if self.state == 'import':
+            if content == "\n" and self.parens <= 0:
+                self.state = None
+                self.parens = 0
+                self.aliases = 0
+        else:
+            self.state = None
+
+    def _process_other(self, token, content):
+        if self.state == 'import':
+            self.aliases += 1
+        else:
+            self.state = None
+
+    def _process_misc(self, token, content):
+        self.state = None

packages/wakatime/dependencies/php.py

@@ -61,10 +61,10 @@ class PhpParser(TokenParser):
 
     def _process_literal_string(self, token, content):
         if self.state == 'include':
-            if content != '"':
+            if content != '"' and content != "'":
                 content = content.strip()
                 if u(token) == 'Token.Literal.String.Double':
-                    content = u('"{0}"').format(content)
+                    content = u("'{0}'").format(content)
                 self.append(content)
                 self.state = None
 

packages/wakatime/dependencies/python.py

@@ -18,7 +18,9 @@ class PythonParser(TokenParser):
     nonpackage = False
     exclude = [
         r'^os$',
+        r'^sys$',
         r'^sys\.',
+        r'^__future__$',
     ]
 
     def parse(self):
@@ -48,9 +50,7 @@ class PythonParser(TokenParser):
                 self._process_import(token, content)
 
     def _process_operator(self, token, content):
-        if self.state is not None:
-            if content == '.':
-                self.nonpackage = True
+        pass
 
     def _process_punctuation(self, token, content):
         if content == '(':
@@ -73,8 +73,6 @@ class PythonParser(TokenParser):
             if self.state == 'from':
                 self.append(content, truncate=True, truncate_to=1)
                 self.state = 'from-2'
-            elif self.state == 'from-2' and content != 'import':
-                self.append(content, truncate=True, truncate_to=1)
             elif self.state == 'import':
                 self.append(content, truncate=True, truncate_to=1)
                 self.state = 'import-2'

packages/wakatime/dependencies/templates.py

@@ -69,30 +69,9 @@ KEYWORDS = [
 ]
 
 
-class LassoJavascriptParser(TokenParser):
-
-    def parse(self):
-        for index, token, content in self.tokens:
-            self._process_token(token, content)
-        return self.dependencies
-
-    def _process_token(self, token, content):
-        if u(token) == 'Token.Name.Other':
-            self._process_name(token, content)
-        elif u(token) == 'Token.Literal.String.Single' or u(token) == 'Token.Literal.String.Double':
-            self._process_literal_string(token, content)
-
-    def _process_name(self, token, content):
-        if content.lower() in KEYWORDS:
-            self.append(content.lower())
-
-    def _process_literal_string(self, token, content):
-        if 'famous/core/' in content.strip('"').strip("'"):
-            self.append('famous')
-
-
 class HtmlDjangoParser(TokenParser):
     tags = []
+    opening_tag = False
     getting_attrs = False
     current_attr = None
     current_attr_value = None
@@ -103,7 +82,9 @@ class HtmlDjangoParser(TokenParser):
         return self.dependencies
 
     def _process_token(self, token, content):
-        if u(token) == 'Token.Name.Tag':
+        if u(token) == 'Token.Punctuation':
+            self._process_punctuation(token, content)
+        elif u(token) == 'Token.Name.Tag':
             self._process_tag(token, content)
         elif u(token) == 'Token.Literal.String':
             self._process_string(token, content)
@@ -114,26 +95,30 @@ class HtmlDjangoParser(TokenParser):
     def current_tag(self):
         return None if len(self.tags) == 0 else self.tags[0]
 
-    def _process_tag(self, token, content):
+    def _process_punctuation(self, token, content):
         if content.startswith('</') or content.startswith('/'):
             try:
                 self.tags.pop(0)
             except IndexError:
                 # ignore errors from malformed markup
                 pass
+            self.opening_tag = False
             self.getting_attrs = False
         elif content.startswith('<'):
+            self.opening_tag = True
+        elif content.startswith('>'):
+            self.opening_tag = False
+            self.getting_attrs = False
+
+    def _process_tag(self, token, content):
+        if self.opening_tag:
             self.tags.insert(0, content.replace('<', '', 1).strip().lower())
             self.getting_attrs = True
-        elif content.startswith('>'):
-            self.getting_attrs = False
         self.current_attr = None
 
     def _process_attribute(self, token, content):
         if self.getting_attrs:
             self.current_attr = content.lower().strip('=')
-        else:
-            self.current_attr = None
         self.current_attr_value = None
 
     def _process_string(self, token, content):
@@ -156,8 +141,6 @@ class HtmlDjangoParser(TokenParser):
             elif content.startswith('"') or content.startswith("'"):
                 if self.current_attr_value is None:
                     self.current_attr_value = content
-                else:
-                    self.current_attr_value += content
 
 
 class VelocityHtmlParser(HtmlDjangoParser):

packages/wakatime/languages/default.json

@@ -0,0 +1,80 @@
+{
+  "ActionScript": "ActionScript",
+  "ApacheConf": "ApacheConf",
+  "AppleScript": "AppleScript",
+  "ASP": "ASP",
+  "Assembly": "Assembly",
+  "Awk": "Awk",
+  "Bash": "Bash",
+  "Basic": "Basic",
+  "BrightScript": "BrightScript",
+  "C": "C",
+  "C#": "C#",
+  "C++": "C++",
+  "Clojure": "Clojure", 
+  "Cocoa": "Cocoa",
+  "CoffeeScript": "CoffeeScript",
+  "ColdFusion": "ColdFusion",
+  "Common Lisp": "Common Lisp",
+  "CSHTML": "CSHTML",
+  "CSS": "CSS",
+  "Dart": "Dart",
+  "Delphi": "Delphi",
+  "Elixir": "Elixir",
+  "Elm": "Elm",
+  "Emacs Lisp": "Emacs Lisp",
+  "Erlang": "Erlang",
+  "F#": "F#",
+  "Fortran": "Fortran",
+  "Go": "Go",
+  "Gous": "Gosu",
+  "Groovy": "Groovy",
+  "Haml": "Haml",
+  "HaXe": "HaXe",
+  "Haskell": "Haskell",
+  "HTML": "HTML",
+  "INI": "INI",
+  "Jade": "Jade",
+  "Java": "Java",
+  "JavaScript": "JavaScript",
+  "JSON": "JSON",
+  "JSX": "JSX",
+  "Kotlin": "Kotlin",
+  "LESS": "LESS",
+  "Lua": "Lua",
+  "Markdown": "Markdown",
+  "Matlab": "Matlab",
+  "Mustache": "Mustache",
+  "OCaml": "OCaml",
+  "Objective-C": "Objective-C",
+  "Objective-C++": "Objective-C++",
+  "Objective-J": "Objective-J",
+  "Perl": "Perl",
+  "PHP": "PHP",
+  "PowerShell": "PowerShell",
+  "Prolog": "Prolog",
+  "Puppet": "Puppet",
+  "Python": "Python",
+  "R": "R",
+  "reStructuredText": "reStructuredText",
+  "Ruby": "Ruby",
+  "Rust": "Rust",
+  "Sass": "Sass",
+  "Scala": "Scala",
+  "Scheme": "Scheme",
+  "SCSS": "SCSS",
+  "Shell": "Shell",
+  "Slim": "Slim",
+  "Smalltalk": "Smalltalk",
+  "SQL": "SQL",
+  "Swift": "Swift",
+  "Text": "Text",
+  "Turing": "Turing",
+  "Twig": "Twig",
+  "TypeScript": "TypeScript",
+  "VB.net": "VB.net",
+  "VimL": "VimL",
+  "XAML": "XAML",
+  "XML": "XML",
+  "YAML": "YAML"
+}

packages/wakatime/languages/vim.json

@@ -0,0 +1,531 @@
+{
+  "a2ps": null,
+  "a65": "Assembly",
+  "aap": null,
+  "abap": null,
+  "abaqus": null,
+  "abc": null,
+  "abel": null,
+  "acedb": null,
+  "ada": null,
+  "aflex": null,
+  "ahdl": null,
+  "alsaconf": null,
+  "amiga": null,
+  "aml": null,
+  "ampl": null,
+  "ant": null,
+  "antlr": null,
+  "apache": null,
+  "apachestyle": null,
+  "arch": null,
+  "art": null,
+  "asm": "Assembly",
+  "asm68k": "Assembly",
+  "asmh8300": "Assembly",
+  "asn": null,
+  "aspperl": null,
+  "aspvbs": null,
+  "asterisk": null,
+  "asteriskvm": null,
+  "atlas": null,
+  "autohotkey": null,
+  "autoit": null,
+  "automake": null,
+  "ave": null,
+  "awk": null,
+  "ayacc": null,
+  "b": null,
+  "baan": null,
+  "basic": "Basic",
+  "bc": null,
+  "bdf": null,
+  "bib": null,
+  "bindzone": null,
+  "blank": null,
+  "bst": null,
+  "btm": null,
+  "bzr": null,
+  "c": "C",
+  "cabal": null,
+  "calendar": null,
+  "catalog": null,
+  "cdl": null,
+  "cdrdaoconf": null,
+  "cdrtoc": null,
+  "cf": null,
+  "cfg": null,
+  "ch": null,
+  "chaiscript": null,
+  "change": null,
+  "changelog": null,
+  "chaskell": null,
+  "cheetah": null,
+  "chill": null,
+  "chordpro": null,
+  "cl": null,
+  "clean": null,
+  "clipper": null,
+  "cmake": null,
+  "cmusrc": null,
+  "cobol": null,
+  "coco": null,
+  "conaryrecipe": null,
+  "conf": null,
+  "config": null,
+  "context": null,
+  "cpp": "C++",
+  "crm": null,
+  "crontab": "Crontab",
+  "cs": "C#",
+  "csc": null,
+  "csh": null,
+  "csp": null,
+  "css": null,
+  "cterm": null,
+  "ctrlh": null,
+  "cucumber": null,
+  "cuda": null,
+  "cupl": null,
+  "cuplsim": null,
+  "cvs": null,
+  "cvsrc": null,
+  "cweb": null,
+  "cynlib": null,
+  "cynpp": null,
+  "d": null,
+  "datascript": null,
+  "dcd": null,
+  "dcl": null,
+  "debchangelog": null,
+  "debcontrol": null,
+  "debsources": null,
+  "def": null,
+  "denyhosts": null,
+  "desc": null,
+  "desktop": null,
+  "dictconf": null,
+  "dictdconf": null,
+  "diff": null,
+  "dircolors": null,
+  "diva": null,
+  "django": null,
+  "dns": null,
+  "docbk": null,
+  "docbksgml": null,
+  "docbkxml": null,
+  "dosbatch": null,
+  "dosini": null,
+  "dot": null,
+  "doxygen": null,
+  "dracula": null,
+  "dsl": null,
+  "dtd": null,
+  "dtml": null,
+  "dtrace": null,
+  "dylan": null,
+  "dylanintr": null,
+  "dylanlid": null,
+  "ecd": null,
+  "edif": null,
+  "eiffel": null,
+  "elf": null,
+  "elinks": null,
+  "elmfilt": null,
+  "erlang": null,
+  "eruby": null,
+  "esmtprc": null,
+  "esqlc": null,
+  "esterel": null,
+  "eterm": null,
+  "eviews": null,
+  "exim": null,
+  "expect": null,
+  "exports": null,
+  "fan": null,
+  "fasm": null,
+  "fdcc": null,
+  "fetchmail": null,
+  "fgl": null,
+  "flexwiki": null,
+  "focexec": null,
+  "form": null,
+  "forth": null,
+  "fortran": null,
+  "foxpro": null,
+  "framescript": null,
+  "freebasic": null,
+  "fstab": null,
+  "fvwm": null,
+  "fvwm2m4": null,
+  "gdb": null,
+  "gdmo": null,
+  "gedcom": null,
+  "git": null,
+  "gitcommit": null,
+  "gitconfig": null,
+  "gitrebase": null,
+  "gitsendemail": null,
+  "gkrellmrc": null,
+  "gnuplot": null,
+  "gp": null,
+  "gpg": null,
+  "grads": null,
+  "gretl": null,
+  "groff": null,
+  "groovy": null,
+  "group": null,
+  "grub": null,
+  "gsp": null,
+  "gtkrc": null,
+  "haml": "Haml",
+  "hamster": null,
+  "haskell": "Haskell",
+  "haste": null,
+  "hastepreproc": null,
+  "hb": null,
+  "help": null,
+  "hercules": null,
+  "hex": null,
+  "hog": null,
+  "hostconf": null,
+  "html": "HTML",
+  "htmlcheetah": "HTML",
+  "htmldjango": "HTML",
+  "htmlm4": "HTML",
+  "htmlos": null,
+  "ia64": null,
+  "ibasic": null,
+  "icemenu": null,
+  "icon": null,
+  "idl": null,
+  "idlang": null,
+  "indent": null,
+  "inform": null,
+  "initex": null,
+  "initng": null,
+  "inittab": null,
+  "ipfilter": null,
+  "ishd": null,
+  "iss": null,
+  "ist": null,
+  "jal": null,
+  "jam": null,
+  "jargon": null,
+  "java": "Java",
+  "javacc": null,
+  "javascript": "JavaScript",
+  "jess": null,
+  "jgraph": null,
+  "jproperties": null,
+  "jsp": null,
+  "kconfig": null,
+  "kix": null,
+  "kscript": null,
+  "kwt": null,
+  "lace": null,
+  "latte": null,
+  "ld": null,
+  "ldapconf": null,
+  "ldif": null,
+  "lex": null,
+  "lftp": null,
+  "lhaskell": "Haskell",
+  "libao": null,
+  "lifelines": null,
+  "lilo": null,
+  "limits": null,
+  "liquid": null,
+  "lisp": null,
+  "lite": null,
+  "litestep": null,
+  "loginaccess": null,
+  "logindefs": null,
+  "logtalk": null,
+  "lotos": null,
+  "lout": null,
+  "lpc": null,
+  "lprolog": null,
+  "lscript": null,
+  "lsl": null,
+  "lss": null,
+  "lua": null,
+  "lynx": null,
+  "m4": null,
+  "mail": null,
+  "mailaliases": null,
+  "mailcap": null,
+  "make": null,
+  "man": null,
+  "manconf": null,
+  "manual": null,
+  "maple": null,
+  "markdown": "Markdown",
+  "masm": null,
+  "mason": null,
+  "master": null,
+  "matlab": null,
+  "maxima": null,
+  "mel": null,
+  "messages": null,
+  "mf": null,
+  "mgl": null,
+  "mgp": null,
+  "mib": null,
+  "mma": null,
+  "mmix": null,
+  "mmp": null,
+  "modconf": null,
+  "model": null,
+  "modsim3": null,
+  "modula2": null,
+  "modula3": null,
+  "monk": null,
+  "moo": null,
+  "mp": null,
+  "mplayerconf": null,
+  "mrxvtrc": null,
+  "msidl": null,
+  "msmessages": null,
+  "msql": null,
+  "mupad": null,
+  "mush": null,
+  "muttrc": null,
+  "mysql": null,
+  "named": null,
+  "nanorc": null,
+  "nasm": null,
+  "nastran": null,
+  "natural": null,
+  "ncf": null,
+  "netrc": null,
+  "netrw": null,
+  "nosyntax": null,
+  "nqc": null,
+  "nroff": null,
+  "nsis": null,
+  "obj": null,
+  "objc": "Objective-C",
+  "objcpp": "Objective-C++",
+  "ocaml": "OCaml",
+  "occam": null,
+  "omnimark": null,
+  "openroad": null,
+  "opl": null,
+  "ora": null,
+  "pamconf": null,
+  "papp": null,
+  "pascal": null,
+  "passwd": null,
+  "pcap": null,
+  "pccts": null,
+  "pdf": null,
+  "perl": "Perl",
+  "perl6": "Perl",
+  "pf": null,
+  "pfmain": null,
+  "php": "PHP",
+  "phtml": "PHP",
+  "pic": null,
+  "pike": null,
+  "pilrc": null,
+  "pine": null,
+  "pinfo": null,
+  "plaintex": null,
+  "plm": null,
+  "plp": null,
+  "plsql": null,
+  "po": null,
+  "pod": null,
+  "postscr": null,
+  "pov": null,
+  "povini": null,
+  "ppd": null,
+  "ppwiz": null,
+  "prescribe": null,
+  "privoxy": null,
+  "procmail": null,
+  "progress": null,
+  "prolog": "Prolog",
+  "promela": null,
+  "protocols": null,
+  "psf": null,
+  "ptcap": null,
+  "purifylog": null,
+  "pyrex": null,
+  "python": "Python",
+  "qf": null,
+  "quake": null,
+  "r": "R",
+  "racc": null,
+  "radiance": null,
+  "ratpoison": null,
+  "rc": null,
+  "rcs": null,
+  "rcslog": null,
+  "readline": null,
+  "rebol": null,
+  "registry": null,
+  "remind": null,
+  "resolv": null,
+  "reva": null,
+  "rexx": null,
+  "rhelp": null,
+  "rib": null,
+  "rnc": null,
+  "rnoweb": null,
+  "robots": null,
+  "rpcgen": null,
+  "rpl": null,
+  "rst": null,
+  "rtf": null,
+  "ruby": "Ruby",
+  "samba": null,
+  "sas": null,
+  "sass": "Sass",
+  "sather": null,
+  "scheme": "Scheme",
+  "scilab": null,
+  "screen": null,
+  "scss": "SCSS",
+  "sd": null,
+  "sdc": null,
+  "sdl": null,
+  "sed": null,
+  "sendpr": null,
+  "sensors": null,
+  "services": null,
+  "setserial": null,
+  "sgml": null,
+  "sgmldecl": null,
+  "sgmllnx": null,
+  "sh": null,
+  "sicad": null,
+  "sieve": null,
+  "simula": null,
+  "sinda": null,
+  "sindacmp": null,
+  "sindaout": null,
+  "sisu": null,
+  "skill": "SKILL",
+  "sl": null,
+  "slang": null,
+  "slice": null,
+  "slpconf": null,
+  "slpreg": null,
+  "slpspi": null,
+  "slrnrc": null,
+  "slrnsc": null,
+  "sm": null,
+  "smarty": null,
+  "smcl": null,
+  "smil": null,
+  "smith": null,
+  "sml": null,
+  "snnsnet": null,
+  "snnspat": null,
+  "snnsres": null,
+  "snobol4": null,
+  "spec": null,
+  "specman": null,
+  "spice": null,
+  "splint": null,
+  "spup": null,
+  "spyce": null,
+  "sql": null,
+  "sqlanywhere": null,
+  "sqlforms": null,
+  "sqlinformix": null,
+  "sqlj": null,
+  "sqloracle": null,
+  "sqr": null,
+  "squid": null,
+  "sshconfig": null,
+  "sshdconfig": null,
+  "st": null,
+  "stata": null,
+  "stp": null,
+  "strace": null,
+  "sudoers": null,
+  "svg": null,
+  "svn": null,
+  "syncolor": null,
+  "synload": null,
+  "syntax": null,
+  "sysctl": null,
+  "tads": null,
+  "tags": null,
+  "tak": null,
+  "takcmp": null,
+  "takout": null,
+  "tar": null,
+  "taskdata": null,
+  "taskedit": null,
+  "tasm": null,
+  "tcl": null,
+  "tcsh": null,
+  "terminfo": null,
+  "tex": null,
+  "texinfo": null,
+  "texmf": null,
+  "tf": null,
+  "tidy": null,
+  "tilde": null,
+  "tli": null,
+  "tpp": null,
+  "trasys": null,
+  "trustees": null,
+  "tsalt": null,
+  "tsscl": null,
+  "tssgm": null,
+  "tssop": null,
+  "uc": null,
+  "udevconf": null,
+  "udevperm": null,
+  "udevrules": null,
+  "uil": null,
+  "updatedb": null,
+  "valgrind": null,
+  "vb": "VB.net",
+  "vera": null,
+  "verilog": null,
+  "verilogams": null,
+  "vgrindefs": null,
+  "vhdl": null,
+  "vim": "VimL",
+  "viminfo": null,
+  "virata": null,
+  "vmasm": null,
+  "voscm": null,
+  "vrml": null,
+  "vsejcl": null,
+  "wdiff": null,
+  "web": null,
+  "webmacro": null,
+  "wget": null,
+  "winbatch": null,
+  "wml": null,
+  "wsh": null,
+  "wsml": null,
+  "wvdial": null,
+  "xbl": null,
+  "xdefaults": null,
+  "xf86conf": null,
+  "xhtml": "HTML",
+  "xinetd": null,
+  "xkb": null,
+  "xmath": null,
+  "xml": "XML",
+  "xmodmap": null,
+  "xpm": null,
+  "xpm2": null,
+  "xquery": null,
+  "xs": null,
+  "xsd": null,
+  "xslt": null,
+  "xxd": null,
+  "yacc": null,
+  "yaml": "YAML",
+  "z8a": null,
+  "zsh": null
+}

packages/wakatime/logger.py

@@ -11,9 +11,10 @@
 
 import logging
 import os
-import sys
+import traceback
 
 from .compat import u
+from .packages.requests.packages import urllib3
 try:
     from collections import OrderedDict  # pragma: nocover
 except ImportError:  # pragma: nocover
@@ -24,26 +25,12 @@ except (ImportError, SyntaxError):  # pragma: nocover
     import json
 
 
-class CustomEncoder(json.JSONEncoder):
-
-    def default(self, obj):
-        if isinstance(obj, bytes):  # pragma: nocover
-            obj = u(obj)
-            return json.dumps(obj)
-        try:  # pragma: nocover
-            encoded = super(CustomEncoder, self).default(obj)
-        except UnicodeDecodeError:  # pragma: nocover
-            obj = u(obj)
-            encoded = super(CustomEncoder, self).default(obj)
-        return encoded
-
-
 class JsonFormatter(logging.Formatter):
 
-    def setup(self, timestamp, isWrite, entity, version, plugin, verbose,
+    def setup(self, timestamp, is_write, entity, version, plugin, verbose,
               warnings=False):
         self.timestamp = timestamp
-        self.isWrite = isWrite
+        self.is_write = is_write
         self.entity = entity
         self.version = version
         self.plugin = plugin
@@ -54,24 +41,25 @@ class JsonFormatter(logging.Formatter):
         data = OrderedDict([
             ('now', self.formatTime(record, self.datefmt)),
         ])
-        data['version'] = self.version
-        data['plugin'] = self.plugin
+        data['version'] = u(self.version)
+        if self.plugin:
+            data['plugin'] = u(self.plugin)
         data['time'] = self.timestamp
         if self.verbose:
-            data['caller'] = record.pathname
+            data['caller'] = u(record.pathname)
             data['lineno'] = record.lineno
-            data['isWrite'] = self.isWrite
-            data['file'] = self.entity
-            if not self.isWrite:
-                del data['isWrite']
+            if self.is_write:
+                data['is_write'] = self.is_write
+            data['file'] = u(self.entity)
         data['level'] = record.levelname
-        data['message'] = record.getMessage() if self.warnings else record.msg
-        if not self.plugin:
-            del data['plugin']
-        return CustomEncoder().encode(data)
+        data['message'] = u(record.getMessage() if self.warnings else record.msg)
+        return json.dumps(data)
 
-    def formatException(self, exc_info):
-        return sys.exec_info[2].format_exc()
+    def traceback(self, lvl=None):
+        logger = logging.getLogger('WakaTime')
+        if not lvl:
+            lvl = logger.getEffectiveLevel()
+        logger.log(lvl, traceback.format_exc())
 
 
 def set_log_level(logger, args):
@@ -82,6 +70,7 @@ def set_log_level(logger, args):
 
 
 def setup_logging(args, version):
+    urllib3.disable_warnings()
     logger = logging.getLogger('WakaTime')
     for handler in logger.handlers:
         logger.removeHandler(handler)
@@ -93,7 +82,7 @@ def setup_logging(args, version):
     formatter = JsonFormatter(datefmt='%Y/%m/%d %H:%M:%S %z')
     formatter.setup(
         timestamp=args.timestamp,
-        isWrite=args.isWrite,
+        is_write=args.is_write,
         entity=args.entity,
         version=version,
         plugin=args.plugin,
@@ -102,10 +91,13 @@ def setup_logging(args, version):
     handler.setFormatter(formatter)
     logger.addHandler(handler)
 
+    # add custom traceback logging method
+    logger.traceback = formatter.traceback
+
     warnings_formatter = JsonFormatter(datefmt='%Y/%m/%d %H:%M:%S %z')
     warnings_formatter.setup(
         timestamp=args.timestamp,
-        isWrite=args.isWrite,
+        is_write=args.is_write,
         entity=args.entity,
         version=version,
         plugin=args.plugin,

packages/wakatime/main.py

@@ -25,14 +25,24 @@ try:
 except ImportError:  # pragma: nocover
     import configparser
 
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'packages'))
+pwd = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.dirname(pwd))
+sys.path.insert(0, os.path.join(pwd, 'packages'))
 
 from .__about__ import __version__
 from .compat import u, open, is_py3
+from .constants import (
+    API_ERROR,
+    AUTH_ERROR,
+    CONFIG_FILE_PARSE_ERROR,
+    SUCCESS,
+    UNKNOWN_ERROR,
+    MALFORMED_HEARTBEAT_ERROR,
+)
 from .logger import setup_logging
 from .offlinequeue import Queue
 from .packages import argparse
+from .packages import requests
 from .packages.requests.exceptions import RequestException
 from .project import get_project_info
 from .session_cache import SessionCache
@@ -41,10 +51,7 @@ try:
     from .packages import simplejson as json  # pragma: nocover
 except (ImportError, SyntaxError):  # pragma: nocover
     import json
-try:
-    from .packages import tzlocal
-except:  # pragma: nocover
-    from .packages import tzlocal3 as tzlocal
+from .packages import tzlocal
 
 
 log = logging.getLogger('WakaTime')
@@ -67,6 +74,12 @@ def parseConfigFile(configFile=None):
     at ~/.wakatime.cfg.
     """
 
+    # get config file location from ENV
+    home = os.environ.get('WAKATIME_HOME')
+    if not configFile and home:
+        configFile = os.path.join(os.path.expanduser(home), '.wakatime.cfg')
+
+    # use default config file location
     if not configFile:
         configFile = os.path.join(os.path.expanduser('~'), '.wakatime.cfg')
 
@@ -79,7 +92,8 @@ def parseConfigFile(configFile=None):
                 print(traceback.format_exc())
                 return None
     except IOError:
-        print(u('Error: Could not read from config file {0}').format(u(configFile)))
+        sys.stderr.write(u("Error: Could not read from config file {0}\n").format(u(configFile)))
+        raise SystemExit(CONFIG_FILE_PARSE_ERROR)
     return configs
 
 
@@ -95,13 +109,13 @@ def parseArguments():
     parser.add_argument('--entity', dest='entity', metavar='FILE',
             action=FileAction,
             help='absolute path to file for the heartbeat; can also be a '+
-                 'url, domain, or app when --entitytype is not file')
+                 'url, domain, or app when --entity-type is not file')
     parser.add_argument('--file', dest='file', action=FileAction,
             help=argparse.SUPPRESS)
     parser.add_argument('--key', dest='key',
             help='your wakatime api key; uses api_key from '+
-                '~/.wakatime.conf by default')
-    parser.add_argument('--write', dest='isWrite',
+                '~/.wakatime.cfg by default')
+    parser.add_argument('--write', dest='is_write',
             action='store_true',
             help='when set, tells api this heartbeat was triggered from '+
                  'writing to a file')
@@ -116,17 +130,24 @@ def parseArguments():
             help='optional line number; current line being edited')
     parser.add_argument('--cursorpos', dest='cursorpos',
             help='optional cursor position in the current file')
-    parser.add_argument('--entitytype', dest='entity_type',
+    parser.add_argument('--entity-type', dest='entity_type',
             help='entity type for this heartbeat. can be one of "file", '+
-                 '"url", "domain", or "app"; defaults to file.')
+                 '"domain", or "app"; defaults to file.')
     parser.add_argument('--proxy', dest='proxy',
-                        help='optional https proxy url; for example: '+
-                        'https://user:pass@localhost:8080')
+                        help='optional proxy configuration. Supports HTTPS '+
+                        'and SOCKS proxies. For example: '+
+                        'https://user:pass@host:port or '+
+                        'socks5://user:pass@host:port')
     parser.add_argument('--project', dest='project',
             help='optional project name')
     parser.add_argument('--alternate-project', dest='alternate_project',
-            help='optional alternate project name; auto-discovered project takes priority')
-    parser.add_argument('--hostname', dest='hostname', help='hostname of current machine.')
+            help='optional alternate project name; auto-discovered project '+
+                 'takes priority')
+    parser.add_argument('--alternate-language', dest='alternate_language',
+            help='optional alternate language name; auto-detected language'+
+                 'takes priority')
+    parser.add_argument('--hostname', dest='hostname', help='hostname of '+
+                        'current machine.')
     parser.add_argument('--disableoffline', dest='offline',
             action='store_false',
             help='disables offline time logging instead of queuing logged time')
@@ -142,14 +163,18 @@ def parseArguments():
                  'POSIX regex syntax; can be used more than once')
     parser.add_argument('--ignore', dest='ignore', action='append',
             help=argparse.SUPPRESS)
+    parser.add_argument('--extra-heartbeats', dest='extra_heartbeats',
+            action='store_true',
+            help='reads extra heartbeats from STDIN as a JSON array until EOF')
     parser.add_argument('--logfile', dest='logfile',
             help='defaults to ~/.wakatime.log')
     parser.add_argument('--apiurl', dest='api_url',
             help='heartbeats api url; for debugging with a local server')
     parser.add_argument('--timeout', dest='timeout', type=int,
-            help='number of seconds to wait when sending heartbeats to api')
+            help='number of seconds to wait when sending heartbeats to api; '+
+                 'defaults to 60 seconds')
     parser.add_argument('--config', dest='config',
-            help='defaults to ~/.wakatime.conf')
+            help='defaults to ~/.wakatime.cfg')
     parser.add_argument('--verbose', dest='verbose', action='store_true',
             help='turns on debug messages in log file')
     parser.add_argument('--version', action='version', version=__version__)
@@ -167,6 +192,9 @@ def parseArguments():
         return args, configs
 
     # update args from configs
+    if not args.hostname:
+        if configs.has_option('settings', 'hostname'):
+            args.hostname = configs.get('settings', 'hostname')
     if not args.key:
         default_key = None
         if configs.has_option('settings', 'api_key'):
@@ -176,9 +204,10 @@ def parseArguments():
         if default_key:
             args.key = default_key
         else:
-            parser.error('Missing api key')
-    if not args.entity_type:
-        args.entity_type = 'file'
+            try:
+                parser.error('Missing api key')
+            except SystemExit:
+                raise SystemExit(AUTH_ERROR)
     if not args.entity:
         if args.file:
             args.entity = args.file
@@ -209,10 +238,23 @@ def parseArguments():
                     args.include.append(pattern)
         except TypeError:  # pragma: nocover
             pass
+    if args.hidefilenames:
+        args.hidefilenames = ['.*']
+    else:
+        args.hidefilenames = []
+        if configs.has_option('settings', 'hidefilenames'):
+            option = configs.get('settings', 'hidefilenames')
+            if option.strip().lower() == 'true':
+                args.hidefilenames = ['.*']
+            elif option.strip().lower() != 'false':
+                try:
+                    for pattern in option.split("\n"):
+                        if pattern.strip() != '':
+                            args.hidefilenames.append(pattern)
+                except TypeError:
+                    pass
     if args.offline and configs.has_option('settings', 'offline'):
         args.offline = configs.getboolean('settings', 'offline')
-    if not args.hidefilenames and configs.has_option('settings', 'hidefilenames'):
-        args.hidefilenames = configs.getboolean('settings', 'hidefilenames')
     if not args.proxy and configs.has_option('settings', 'proxy'):
         args.proxy = configs.get('settings', 'proxy')
     if not args.verbose and configs.has_option('settings', 'verbose'):
@@ -221,6 +263,9 @@ def parseArguments():
         args.verbose = configs.getboolean('settings', 'debug')
     if not args.logfile and configs.has_option('settings', 'logfile'):
         args.logfile = configs.get('settings', 'logfile')
+    if not args.logfile and os.environ.get('WAKATIME_HOME'):
+        home = os.environ.get('WAKATIME_HOME')
+        args.logfile = os.path.join(os.path.expanduser(home), '.wakatime.log')
     if not args.api_url and configs.has_option('settings', 'api_url'):
         args.api_url = configs.get('settings', 'api_url')
     if not args.timeout and configs.has_option('settings', 'timeout'):
@@ -234,32 +279,26 @@ def parseArguments():
 
 def should_exclude(entity, include, exclude):
     if entity is not None and entity.strip() != '':
-        try:
-            for pattern in include:
-                try:
-                    compiled = re.compile(pattern, re.IGNORECASE)
-                    if compiled.search(entity):
-                        return False
-                except re.error as ex:
-                    log.warning(u('Regex error ({msg}) for include pattern: {pattern}').format(
-                        msg=u(ex),
-                        pattern=u(pattern),
-                    ))
-        except TypeError:  # pragma: nocover
-            pass
-        try:
-            for pattern in exclude:
-                try:
-                    compiled = re.compile(pattern, re.IGNORECASE)
-                    if compiled.search(entity):
-                        return pattern
-                except re.error as ex:
-                    log.warning(u('Regex error ({msg}) for exclude pattern: {pattern}').format(
-                        msg=u(ex),
-                        pattern=u(pattern),
-                    ))
-        except TypeError:  # pragma: nocover
-            pass
+        for pattern in include:
+            try:
+                compiled = re.compile(pattern, re.IGNORECASE)
+                if compiled.search(entity):
+                    return False
+            except re.error as ex:
+                log.warning(u('Regex error ({msg}) for include pattern: {pattern}').format(
+                    msg=u(ex),
+                    pattern=u(pattern),
+                ))
+        for pattern in exclude:
+            try:
+                compiled = re.compile(pattern, re.IGNORECASE)
+                if compiled.search(entity):
+                    return pattern
+            except re.error as ex:
+                log.warning(u('Regex error ({msg}) for exclude pattern: {pattern}').format(
+                    msg=u(ex),
+                    pattern=u(pattern),
+                ))
     return False
 
 
@@ -283,16 +322,20 @@ def get_user_agent(plugin):
     return user_agent
 
 
-def send_heartbeat(project=None, branch=None, hostname=None, stats={}, key=None, entity=None,
-        timestamp=None, isWrite=None, plugin=None, offline=None, entity_type='file',
-        hidefilenames=None, proxy=None, api_url=None, timeout=None, **kwargs):
+def send_heartbeat(project=None, branch=None, hostname=None, stats={}, key=None,
+                   entity=None, timestamp=None, is_write=None, plugin=None,
+                   offline=None, entity_type='file', hidefilenames=None,
+                   proxy=None, api_url=None, timeout=None, **kwargs):
     """Sends heartbeat as POST request to WakaTime api server.
+
+    Returns `SUCCESS` when heartbeat was sent, otherwise returns an
+    error code constant.
     """
 
     if not api_url:
-        api_url = 'https://wakatime.com/api/v1/heartbeats'
+        api_url = 'https://api.wakatime.com/api/v1/heartbeats'
     if not timeout:
-        timeout = 30
+        timeout = 60
     log.debug('Sending heartbeat to api at %s' % api_url)
     data = {
         'time': timestamp,
@@ -300,8 +343,18 @@ def send_heartbeat(project=None, branch=None, hostname=None, stats={}, key=None,
         'type': entity_type,
     }
     if hidefilenames and entity is not None and entity_type == 'file':
-        extension = u(os.path.splitext(data['entity'])[1])
-        data['entity'] = u('HIDDEN{0}').format(extension)
+        for pattern in hidefilenames:
+            try:
+                compiled = re.compile(pattern, re.IGNORECASE)
+                if compiled.search(entity):
+                    extension = u(os.path.splitext(data['entity'])[1])
+                    data['entity'] = u('HIDDEN{0}').format(extension)
+                    break
+            except re.error as ex:
+                log.warning(u('Regex error ({msg}) for include pattern: {pattern}').format(
+                    msg=u(ex),
+                    pattern=u(pattern),
+                ))
     if stats.get('lines'):
         data['lines'] = stats['lines']
     if stats.get('language'):
@@ -312,8 +365,8 @@ def send_heartbeat(project=None, branch=None, hostname=None, stats={}, key=None,
         data['lineno'] = stats['lineno']
     if stats.get('cursorpos'):
         data['cursorpos'] = stats['cursorpos']
-    if isWrite:
-        data['is_write'] = isWrite
+    if is_write:
+        data['is_write'] = is_write
     if project:
         data['project'] = project
     if branch:
@@ -331,7 +384,7 @@ def send_heartbeat(project=None, branch=None, hostname=None, stats={}, key=None,
         'Authorization': auth,
     }
     if hostname:
-        headers['X-Machine-Name'] = hostname
+        headers['X-Machine-Name'] = u(hostname).encode('utf-8')
     proxies = {}
     if proxy:
         proxies['https'] = proxy
@@ -342,7 +395,7 @@ def send_heartbeat(project=None, branch=None, hostname=None, stats={}, key=None,
     except:
         tz = None
     if tz:
-        headers['TimeZone'] = u(tz.zone)
+        headers['TimeZone'] = u(tz.zone).encode('utf-8')
 
     session_cache = SessionCache()
     session = session_cache.get()
@@ -351,7 +404,7 @@ def send_heartbeat(project=None, branch=None, hostname=None, stats={}, key=None,
     response = None
     try:
         response = session.post(api_url, data=request_body, headers=headers,
-                                 proxies=proxies, timeout=timeout)
+                                proxies=proxies, timeout=timeout)
     except RequestException:
         exception_data = {
             sys.exc_info()[0].__name__: u(sys.exc_info()[1]),
@@ -365,41 +418,145 @@ def send_heartbeat(project=None, branch=None, hostname=None, stats={}, key=None,
                 log.warn(exception_data)
         else:
             log.error(exception_data)
+
+    except:  # delete cached session when requests raises unknown exception
+        exception_data = {
+            sys.exc_info()[0].__name__: u(sys.exc_info()[1]),
+            'traceback': traceback.format_exc(),
+        }
+        if offline:
+            queue = Queue()
+            queue.push(data, json.dumps(stats), plugin)
+            log.warn(exception_data)
+        session_cache.delete()
+
     else:
-        response_code = response.status_code if response is not None else None
-        response_content = response.text if response is not None else None
-        if response_code == 201:
+        code = response.status_code if response is not None else None
+        content = response.text if response is not None else None
+        if code == requests.codes.created or code == requests.codes.accepted:
             log.debug({
-                'response_code': response_code,
+                'response_code': code,
             })
             session_cache.save(session)
-            return True
+            return SUCCESS
         if offline:
-            if response_code != 400:
+            if code != 400:
                 queue = Queue()
                 queue.push(data, json.dumps(stats), plugin)
-                if response_code == 401:
+                if code == 401:
                     log.error({
-                        'response_code': response_code,
-                        'response_content': response_content,
+                        'response_code': code,
+                        'response_content': content,
                     })
+                    session_cache.delete()
+                    return AUTH_ERROR
                 elif log.isEnabledFor(logging.DEBUG):
                     log.warn({
-                        'response_code': response_code,
-                        'response_content': response_content,
+                        'response_code': code,
+                        'response_content': content,
                     })
             else:
                 log.error({
-                    'response_code': response_code,
-                    'response_content': response_content,
+                    'response_code': code,
+                    'response_content': content,
                 })
         else:
             log.error({
-                'response_code': response_code,
-                'response_content': response_content,
+                'response_code': code,
+                'response_content': content,
             })
     session_cache.delete()
-    return False
+    return API_ERROR
+
+
+def sync_offline_heartbeats(args, hostname):
+    """Sends all heartbeats which were cached in the offline Queue."""
+
+    queue = Queue()
+    while True:
+        heartbeat = queue.pop()
+        if heartbeat is None:
+            break
+        status = send_heartbeat(
+            project=heartbeat['project'],
+            entity=heartbeat['entity'],
+            timestamp=heartbeat['time'],
+            branch=heartbeat['branch'],
+            hostname=hostname,
+            stats=json.loads(heartbeat['stats']),
+            key=args.key,
+            is_write=heartbeat['is_write'],
+            plugin=heartbeat['plugin'],
+            offline=args.offline,
+            hidefilenames=args.hidefilenames,
+            entity_type=heartbeat['type'],
+            proxy=args.proxy,
+            api_url=args.api_url,
+            timeout=args.timeout,
+        )
+        if status != SUCCESS:
+            if status == AUTH_ERROR:
+                return AUTH_ERROR
+            break
+    return SUCCESS
+
+
+def format_file_path(filepath):
+    """Formats a path as absolute and with the correct platform separator."""
+
+    try:
+        filepath = os.path.realpath(os.path.abspath(filepath))
+        filepath = re.sub(r'[/\\]', os.path.sep, filepath)
+    except:
+        pass  # pragma: nocover
+    return filepath
+
+
+def process_heartbeat(args, configs, hostname, heartbeat):
+    exclude = should_exclude(heartbeat['entity'], args.include, args.exclude)
+    if exclude is not False:
+        log.debug(u('Skipping because matches exclude pattern: {pattern}').format(
+            pattern=u(exclude),
+        ))
+        return SUCCESS
+
+    if heartbeat.get('entity_type') not in ['file', 'domain', 'app']:
+        heartbeat['entity_type'] = 'file'
+
+    if heartbeat['entity_type'] == 'file':
+        heartbeat['entity'] = format_file_path(heartbeat['entity'])
+
+    if heartbeat['entity_type'] != 'file' or os.path.isfile(heartbeat['entity']):
+
+        stats = get_file_stats(heartbeat['entity'],
+                                entity_type=heartbeat['entity_type'],
+                                lineno=heartbeat.get('lineno'),
+                                cursorpos=heartbeat.get('cursorpos'),
+                                plugin=args.plugin,
+                                alternate_language=heartbeat.get('alternate_language'))
+
+        project = heartbeat.get('project') or heartbeat.get('alternate_project')
+        branch = None
+        if heartbeat['entity_type'] == 'file':
+            project, branch = get_project_info(configs, heartbeat)
+
+        heartbeat['project'] = project
+        heartbeat['branch'] = branch
+        heartbeat['stats'] = stats
+        heartbeat['hostname'] = hostname
+        heartbeat['timeout'] = args.timeout
+        heartbeat['key'] = args.key
+        heartbeat['plugin'] = args.plugin
+        heartbeat['offline'] = args.offline
+        heartbeat['hidefilenames'] = args.hidefilenames
+        heartbeat['proxy'] = args.proxy
+        heartbeat['api_url'] = args.api_url
+
+        return send_heartbeat(**heartbeat)
+
+    else:
+        log.debug('File does not exist; ignoring this heartbeat.')
+        return SUCCESS
 
 
 def execute(argv=None):
@@ -408,63 +565,30 @@ def execute(argv=None):
 
     args, configs = parseArguments()
     if configs is None:
-        return 103 # config file parsing error
+        return CONFIG_FILE_PARSE_ERROR
 
     setup_logging(args, __version__)
 
-    exclude = should_exclude(args.entity, args.include, args.exclude)
-    if exclude is not False:
-        log.debug(u('Skipping because matches exclude pattern: {pattern}').format(
-            pattern=u(exclude),
-        ))
-        return 0
+    try:
 
-    if args.entity_type != 'file' or os.path.isfile(args.entity):
+        hostname = args.hostname or socket.gethostname()
 
-        stats = get_file_stats(args.entity, entity_type=args.entity_type,
-                               lineno=args.lineno, cursorpos=args.cursorpos)
+        heartbeat = vars(args)
+        retval = process_heartbeat(args, configs, hostname, heartbeat)
 
-        project = args.project or args.alternate_project
-        branch = None
-        if args.entity_type == 'file':
-            project, branch = get_project_info(configs, args)
+        if args.extra_heartbeats:
+            try:
+                for heartbeat in json.loads(sys.stdin.readline()):
+                    retval = process_heartbeat(args, configs, hostname, heartbeat)
+            except json.JSONDecodeError:
+                retval = MALFORMED_HEARTBEAT_ERROR
 
-        kwargs = vars(args)
-        kwargs['project'] = project
-        kwargs['branch'] = branch
-        kwargs['stats'] = stats
-        kwargs['hostname'] = args.hostname or socket.gethostname()
-        kwargs['timeout'] = args.timeout
+        if retval == SUCCESS:
+            retval = sync_offline_heartbeats(args, hostname)
 
-        if send_heartbeat(**kwargs):
-            queue = Queue()
-            while True:
-                heartbeat = queue.pop()
-                if heartbeat is None:
-                    break
-                sent = send_heartbeat(
-                    project=heartbeat['project'],
-                    entity=heartbeat['entity'],
-                    timestamp=heartbeat['time'],
-                    branch=heartbeat['branch'],
-                    hostname=kwargs['hostname'],
-                    stats=json.loads(heartbeat['stats']),
-                    key=args.key,
-                    isWrite=heartbeat['is_write'],
-                    plugin=heartbeat['plugin'],
-                    offline=args.offline,
-                    hidefilenames=args.hidefilenames,
-                    entity_type=heartbeat['type'],
-                    proxy=args.proxy,
-                    api_url=args.api_url,
-                    timeout=args.timeout,
-                )
-                if not sent:
-                    break
-            return 0 # success
+        return retval
 
-        return 102 # api error
-
-    else:
-        log.debug('File does not exist; ignoring this heartbeat.')
-        return 0
+    except:
+        log.traceback(logging.ERROR)
+        print(traceback.format_exc())
+        return UNKNOWN_ERROR

packages/wakatime/offlinequeue.py

@@ -12,7 +12,6 @@
 
 import logging
 import os
-import traceback
 from time import sleep
 
 try:
@@ -70,7 +69,7 @@ class Queue(object):
             conn.commit()
             conn.close()
         except sqlite3.Error:
-            log.error(traceback.format_exc())
+            log.traceback()
 
     def pop(self):
         if not HAS_SQL:  # pragma: nocover
@@ -81,7 +80,7 @@ class Queue(object):
         try:
             conn, c = self.connect()
         except sqlite3.Error:
-            log.debug(traceback.format_exc())
+            log.traceback(logging.DEBUG)
             return None
         loop = True
         while loop and tries > -1:
@@ -119,11 +118,11 @@ class Queue(object):
                     }
                 loop = False
             except sqlite3.Error:  # pragma: nocover
-                log.debug(traceback.format_exc())
+                log.traceback(logging.DEBUG)
                 sleep(wait)
                 tries -= 1
         try:
             conn.close()
         except sqlite3.Error:  # pragma: nocover
-            log.debug(traceback.format_exc())
+            log.traceback(logging.DEBUG)
         return heartbeat

packages/wakatime/packages/__init__.py

@@ -0,0 +1,4 @@
+import tzlocal
+from pygments.lexers import get_lexer_by_name, guess_lexer_for_filename
+from pygments.modeline import get_filetype_from_buffer
+from pygments.util import ClassNotFound

packages/wakatime/packages/pygments/__init__.py

@@ -22,20 +22,18 @@
     .. _Pygments tip:
        http://bitbucket.org/birkenfeld/pygments-main/get/tip.zip#egg=Pygments-dev
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
-
-__version__ = '2.0.1'
-__docformat__ = 'restructuredtext'
-
-__all__ = ['lex', 'format', 'highlight']
-
-
 import sys
 
 from pygments.util import StringIO, BytesIO
 
+__version__ = '2.2.0'
+__docformat__ = 'restructuredtext'
+
+__all__ = ['lex', 'format', 'highlight']
+
 
 def lex(code, lexer):
     """
@@ -44,14 +42,15 @@ def lex(code, lexer):
     try:
         return lexer.get_tokens(code)
     except TypeError as err:
-        if isinstance(err.args[0], str) and \
-           'unbound method get_tokens' in err.args[0]:
+        if (isinstance(err.args[0], str) and
+            ('unbound method get_tokens' in err.args[0] or
+             'missing 1 required positional argument' in err.args[0])):
             raise TypeError('lex() argument must be a lexer instance, '
                             'not a class')
         raise
 
 
-def format(tokens, formatter, outfile=None):
+def format(tokens, formatter, outfile=None):  # pylint: disable=redefined-builtin
     """
     Format a tokenlist ``tokens`` with the formatter ``formatter``.
 
@@ -61,15 +60,15 @@ def format(tokens, formatter, outfile=None):
     """
     try:
         if not outfile:
-            #print formatter, 'using', formatter.encoding
-            realoutfile = formatter.encoding and BytesIO() or StringIO()
+            realoutfile = getattr(formatter, 'encoding', None) and BytesIO() or StringIO()
             formatter.format(tokens, realoutfile)
             return realoutfile.getvalue()
         else:
             formatter.format(tokens, outfile)
     except TypeError as err:
-        if isinstance(err.args[0], str) and \
-           'unbound method format' in err.args[0]:
+        if (isinstance(err.args[0], str) and
+            ('unbound method format' in err.args[0] or
+             'missing 1 required positional argument' in err.args[0])):
             raise TypeError('format() argument must be a formatter instance, '
                             'not a class')
         raise
@@ -86,6 +85,6 @@ def highlight(code, lexer, formatter, outfile=None):
     return format(lex(code, lexer), formatter, outfile)
 
 
-if __name__ == '__main__':
+if __name__ == '__main__':  # pragma: no cover
     from pygments.cmdline import main
     sys.exit(main(sys.argv))

packages/wakatime/packages/pygments/cmdline.py

@@ -5,12 +5,13 @@
 
     Command line interface.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 from __future__ import print_function
 
+import os
 import sys
 import getopt
 from textwrap import dedent
@@ -19,18 +20,20 @@ from pygments import __version__, highlight
 from pygments.util import ClassNotFound, OptionError, docstring_headline, \
     guess_decode, guess_decode_from_terminal, terminal_encoding
 from pygments.lexers import get_all_lexers, get_lexer_by_name, guess_lexer, \
-    get_lexer_for_filename, find_lexer_class, TextLexer
+    load_lexer_from_file, get_lexer_for_filename, find_lexer_class_for_filename
+from pygments.lexers.special import TextLexer
 from pygments.formatters.latex import LatexEmbeddedLexer, LatexFormatter
 from pygments.formatters import get_all_formatters, get_formatter_by_name, \
-    get_formatter_for_filename, find_formatter_class, \
-    TerminalFormatter  # pylint:disable-msg=E0611
+    load_formatter_from_file, get_formatter_for_filename, find_formatter_class
+from pygments.formatters.terminal import TerminalFormatter
+from pygments.formatters.terminal256 import Terminal256Formatter
 from pygments.filters import get_all_filters, find_filter_class
 from pygments.styles import get_all_styles, get_style_by_name
 
 
 USAGE = """\
 Usage: %s [-l <lexer> | -g] [-F <filter>[:<options>]] [-f <formatter>]
-          [-O <options>] [-P <option=value>] [-s] [-o <outfile>] [<infile>]
+          [-O <options>] [-P <option=value>] [-s] [-v] [-x] [-o <outfile>] [<infile>]
 
        %s -S <style> -f <formatter> [-a <arg>] [-O <options>] [-P <option=value>]
        %s -L [<which> ...]
@@ -56,6 +59,14 @@ Likewise, <formatter> is a formatter name, and will be guessed from
 the extension of the output file name. If no output file is given,
 the terminal formatter will be used by default.
 
+The additional option -x allows custom lexers and formatters to be
+loaded from a .py file relative to the current working directory. For
+example, ``-l ./customlexer.py -x``. By default, this option expects a
+file with a class named CustomLexer or CustomFormatter; you can also
+specify your own class name with a colon (``-l ./lexer.py:MyLexer``).
+Users should be very careful not to use this option with untrusted files,
+because it will import and run them.
+
 With the -O option, you can give the lexer and formatter a comma-
 separated list of options, e.g. ``-O bg=light,python=cool``.
 
@@ -90,6 +101,9 @@ waiting to process the entire file.  This only works for stdin, and
 is intended for streaming input such as you get from 'tail -f'.
 Example usage: "tail -f sql.log | pygmentize -s -l sql"
 
+The -v option prints a detailed traceback on unhandled exceptions,
+which is useful for debugging and bug reports.
+
 The -h option prints this help.
 The -V option prints the package version.
 """
@@ -100,7 +114,7 @@ def _parse_options(o_strs):
     if not o_strs:
         return opts
     for o_str in o_strs:
-        if not o_str:
+        if not o_str.strip():
             continue
         o_args = o_str.split(',')
         for o_arg in o_args:
@@ -132,7 +146,7 @@ def _parse_filters(f_strs):
 def _print_help(what, name):
     try:
         if what == 'lexer':
-            cls = find_lexer_class(name)
+            cls = get_lexer_by_name(name)
             print("Help on the %s lexer:" % cls.name)
             print(dedent(cls.__doc__))
         elif what == 'formatter':
@@ -143,8 +157,10 @@ def _print_help(what, name):
             cls = find_filter_class(name)
             print("Help on the %s filter:" % name)
             print(dedent(cls.__doc__))
-    except AttributeError:
+        return 0
+    except (AttributeError, ValueError):
         print("%s not found!" % what, file=sys.stderr)
+        return 1
 
 
 def _print_list(what):
@@ -198,19 +214,7 @@ def _print_list(what):
             print("    %s" % docstring_headline(cls))
 
 
-def main(args=sys.argv):
-    """
-    Main command line entry point.
-    """
-    # pylint: disable-msg=R0911,R0912,R0915
-
-    usage = USAGE % ((args[0],) * 6)
-
-    try:
-        popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:hVHgs")
-    except getopt.GetoptError:
-        print(usage, file=sys.stderr)
-        return 2
+def main_inner(popts, args, usage):
     opts = {}
     O_opts = []
     P_opts = []
@@ -229,7 +233,7 @@ def main(args=sys.argv):
         return 0
 
     if opts.pop('-V', None) is not None:
-        print('Pygments version %s, (c) 2006-2014 by Georg Brandl.' % __version__)
+        print('Pygments version %s, (c) 2006-2017 by Georg Brandl.' % __version__)
         return 0
 
     # handle ``pygmentize -L``
@@ -254,13 +258,12 @@ def main(args=sys.argv):
             print(usage, file=sys.stderr)
             return 2
 
-        what, name = args
+        what, name = args  # pylint: disable=unbalanced-tuple-unpacking
         if what not in ('lexer', 'formatter', 'filter'):
             print(usage, file=sys.stderr)
             return 2
 
-        _print_help(what, name)
-        return 0
+        return _print_help(what, name)
 
     # parse -O options
     parsed_opts = _parse_options(O_opts)
@@ -277,19 +280,15 @@ def main(args=sys.argv):
     opts.pop('-P', None)
 
     # encodings
-    inencoding  = parsed_opts.get('inencoding', parsed_opts.get('encoding'))
+    inencoding = parsed_opts.get('inencoding', parsed_opts.get('encoding'))
     outencoding = parsed_opts.get('outencoding', parsed_opts.get('encoding'))
 
     # handle ``pygmentize -N``
     infn = opts.pop('-N', None)
     if infn is not None:
-        try:
-            lexer = get_lexer_for_filename(infn, **parsed_opts)
-        except ClassNotFound as err:
-            lexer = TextLexer()
-        except OptionError as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
+        lexer = find_lexer_class_for_filename(infn)
+        if lexer is None:
+            lexer = TextLexer
 
         print(lexer.aliases[0])
         return 0
@@ -313,12 +312,7 @@ def main(args=sys.argv):
             print(err, file=sys.stderr)
             return 1
 
-        arg = a_opt or ''
-        try:
-            print(fmter.get_style_defs(arg))
-        except Exception as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
+        print(fmter.get_style_defs(a_opt or ''))
         return 0
 
     # if no -S is given, -a is not allowed
@@ -330,14 +324,35 @@ def main(args=sys.argv):
     F_opts = _parse_filters(F_opts)
     opts.pop('-F', None)
 
+    allow_custom_lexer_formatter = False
+    # -x: allow custom (eXternal) lexers and formatters
+    if opts.pop('-x', None) is not None:
+        allow_custom_lexer_formatter = True
+
     # select lexer
-    lexer = opts.pop('-l', None)
-    if lexer:
-        try:
-            lexer = get_lexer_by_name(lexer, **parsed_opts)
-        except (OptionError, ClassNotFound) as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
+    lexer = None
+
+    # given by name?
+    lexername = opts.pop('-l', None)
+    if lexername:
+        # custom lexer, located relative to user's cwd
+        if allow_custom_lexer_formatter and '.py' in lexername:
+            try:
+                if ':' in lexername:
+                    filename, name = lexername.rsplit(':', 1)
+                    lexer = load_lexer_from_file(filename, name,
+                                                 **parsed_opts)
+                else:
+                    lexer = load_lexer_from_file(lexername, **parsed_opts)
+            except ClassNotFound as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
+        else:
+            try:
+                lexer = get_lexer_by_name(lexername, **parsed_opts)
+            except (OptionError, ClassNotFound) as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
 
     # read input code
     code = None
@@ -350,7 +365,7 @@ def main(args=sys.argv):
         if '-s' in opts:
             print('Error: -s option not usable when input file specified',
                   file=sys.stderr)
-            return 1
+            return 2
 
         infn = args[0]
         try:
@@ -396,15 +411,42 @@ def main(args=sys.argv):
             except ClassNotFound:
                 lexer = TextLexer(**parsed_opts)
 
+    else:  # -s option needs a lexer with -l
+        if not lexer:
+            print('Error: when using -s a lexer has to be selected with -l',
+                  file=sys.stderr)
+            return 2
+
+    # process filters
+    for fname, fopts in F_opts:
+        try:
+            lexer.add_filter(fname, **fopts)
+        except ClassNotFound as err:
+            print('Error:', err, file=sys.stderr)
+            return 1
+
     # select formatter
     outfn = opts.pop('-o', None)
     fmter = opts.pop('-f', None)
     if fmter:
-        try:
-            fmter = get_formatter_by_name(fmter, **parsed_opts)
-        except (OptionError, ClassNotFound) as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
+        # custom formatter, located relative to user's cwd
+        if allow_custom_lexer_formatter and '.py' in fmter:
+            try:
+                if ':' in fmter:
+                    file, fmtername = fmter.rsplit(':', 1)
+                    fmter = load_formatter_from_file(file, fmtername,
+                                                     **parsed_opts)
+                else:
+                    fmter = load_formatter_from_file(fmter, **parsed_opts)
+            except ClassNotFound as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
+        else:
+            try:
+                fmter = get_formatter_by_name(fmter, **parsed_opts)
+            except (OptionError, ClassNotFound) as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
 
     if outfn:
         if not fmter:
@@ -420,7 +462,10 @@ def main(args=sys.argv):
             return 1
     else:
         if not fmter:
-            fmter = TerminalFormatter(**parsed_opts)
+            if '256' in os.environ.get('TERM', ''):
+                fmter = Terminal256Formatter(**parsed_opts)
+            else:
+                fmter = TerminalFormatter(**parsed_opts)
         if sys.version_info > (3,):
             # Python 3: we have to use .buffer to get a binary stream
             outfile = sys.stdout.buffer
@@ -438,11 +483,11 @@ def main(args=sys.argv):
 
     # provide coloring under Windows, if possible
     if not outfn and sys.platform in ('win32', 'cygwin') and \
-       fmter.name in ('Terminal', 'Terminal256'):
+       fmter.name in ('Terminal', 'Terminal256'):  # pragma: no cover
         # unfortunately colorama doesn't support binary streams on Py3
         if sys.version_info > (3,):
-            import io
-            outfile = io.TextIOWrapper(outfile, encoding=fmter.encoding)
+            from pygments.util import UnclosingTextIOWrapper
+            outfile = UnclosingTextIOWrapper(outfile, encoding=fmter.encoding)
             fmter.encoding = None
         try:
             import colorama.initialise
@@ -462,39 +507,58 @@ def main(args=sys.argv):
         lexer = LatexEmbeddedLexer(left, right, lexer)
 
     # ... and do it!
+    if '-s' not in opts:
+        # process whole input as per normal...
+        highlight(code, lexer, fmter, outfile)
+        return 0
+    else:
+        # line by line processing of stdin (eg: for 'tail -f')...
+        try:
+            while 1:
+                if sys.version_info > (3,):
+                    # Python 3: we have to use .buffer to get a binary stream
+                    line = sys.stdin.buffer.readline()
+                else:
+                    line = sys.stdin.readline()
+                if not line:
+                    break
+                if not inencoding:
+                    line = guess_decode_from_terminal(line, sys.stdin)[0]
+                highlight(line, lexer, fmter, outfile)
+                if hasattr(outfile, 'flush'):
+                    outfile.flush()
+            return 0
+        except KeyboardInterrupt:  # pragma: no cover
+            return 0
+
+
+def main(args=sys.argv):
+    """
+    Main command line entry point.
+    """
+    usage = USAGE % ((args[0],) * 6)
+
     try:
-        # process filters
-        for fname, fopts in F_opts:
-            lexer.add_filter(fname, **fopts)
-
-        if '-s' not in opts:
-            # process whole input as per normal...
-            highlight(code, lexer, fmter, outfile)
-        else:
-            if not lexer:
-                print('Error: when using -s a lexer has to be selected with -l',
-                      file=sys.stderr)
-                return 1
-            # line by line processing of stdin (eg: for 'tail -f')...
-            try:
-                while 1:
-                    if sys.version_info > (3,):
-                        # Python 3: we have to use .buffer to get a binary stream
-                        line = sys.stdin.buffer.readline()
-                    else:
-                        line = sys.stdin.readline()
-                    if not line:
-                        break
-                    if not inencoding:
-                        line = guess_decode_from_terminal(line, sys.stdin)[0]
-                    highlight(line, lexer, fmter, outfile)
-                    if hasattr(outfile, 'flush'):
-                        outfile.flush()
-            except KeyboardInterrupt:
-                return 0
+        popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:vhVHgsx")
+    except getopt.GetoptError:
+        print(usage, file=sys.stderr)
+        return 2
 
+    try:
+        return main_inner(popts, args, usage)
     except Exception:
-        raise
+        if '-v' in dict(popts):
+            print(file=sys.stderr)
+            print('*' * 65, file=sys.stderr)
+            print('An unhandled exception occurred while highlighting.',
+                  file=sys.stderr)
+            print('Please report the whole traceback to the issue tracker at',
+                  file=sys.stderr)
+            print('<https://bitbucket.org/birkenfeld/pygments-main/issues>.',
+                  file=sys.stderr)
+            print('*' * 65, file=sys.stderr)
+            print(file=sys.stderr)
+            raise
         import traceback
         info = traceback.format_exception(*sys.exc_info())
         msg = info[-1].strip()
@@ -504,6 +568,6 @@ def main(args=sys.argv):
         print(file=sys.stderr)
         print('*** Error while highlighting:', file=sys.stderr)
         print(msg, file=sys.stderr)
+        print('*** If this is a bug you want to report, please rerun with -v.',
+              file=sys.stderr)
         return 1
-
-    return 0

packages/wakatime/packages/pygments/console.py

@@ -5,25 +5,25 @@
 
     Format colored console output.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 esc = "\x1b["
 
 codes = {}
-codes[""]          = ""
-codes["reset"]     = esc + "39;49;00m"
+codes[""] = ""
+codes["reset"] = esc + "39;49;00m"
 
-codes["bold"]      = esc + "01m"
-codes["faint"]     = esc + "02m"
-codes["standout"]  = esc + "03m"
+codes["bold"] = esc + "01m"
+codes["faint"] = esc + "02m"
+codes["standout"] = esc + "03m"
 codes["underline"] = esc + "04m"
-codes["blink"]     = esc + "05m"
-codes["overline"]  = esc + "06m"
+codes["blink"] = esc + "05m"
+codes["overline"] = esc + "06m"
 
-dark_colors  = ["black", "darkred", "darkgreen", "brown", "darkblue",
-                "purple", "teal", "lightgray"]
+dark_colors = ["black", "darkred", "darkgreen", "brown", "darkblue",
+               "purple", "teal", "lightgray"]
 light_colors = ["darkgray", "red", "green", "yellow", "blue",
                 "fuchsia", "turquoise", "white"]
 
@@ -35,10 +35,10 @@ for d, l in zip(dark_colors, light_colors):
 
 del d, l, x
 
-codes["darkteal"]   = codes["turquoise"]
+codes["darkteal"] = codes["turquoise"]
 codes["darkyellow"] = codes["brown"]
-codes["fuscia"]     = codes["fuchsia"]
-codes["white"]      = codes["bold"]
+codes["fuscia"] = codes["fuchsia"]
+codes["white"] = codes["bold"]
 
 
 def reset_color():

packages/wakatime/packages/pygments/filter.py

@@ -5,7 +5,7 @@
 
     Module that implements the default filter.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -29,15 +29,15 @@ def simplefilter(f):
     Decorator that converts a function into a filter::
 
         @simplefilter
-        def lowercase(lexer, stream, options):
+        def lowercase(self, lexer, stream, options):
             for ttype, value in stream:
                 yield ttype, value.lower()
     """
     return type(f.__name__, (FunctionFilter,), {
-                'function':     f,
-                '__module__':   getattr(f, '__module__'),
-                '__doc__':      f.__doc__
-            })
+        '__module__': getattr(f, '__module__'),
+        '__doc__': f.__doc__,
+        'function': f,
+    })
 
 
 class Filter(object):
@@ -69,6 +69,6 @@ class FunctionFilter(Filter):
         Filter.__init__(self, **options)
 
     def filter(self, lexer, stream):
-        # pylint: disable-msg=E1102
+        # pylint: disable=not-callable
         for ttype, value in self.function(lexer, stream, self.options):
             yield ttype, value

packages/wakatime/packages/pygments/filters/__init__.py

@@ -6,7 +6,7 @@
     Module containing filter lookup functions and default
     filters.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/formatter.py

@@ -5,7 +5,7 @@
 
     Base formatter class.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -65,7 +65,7 @@ class Formatter(object):
 
     def __init__(self, **options):
         self.style = _lookup_style(options.get('style', 'default'))
-        self.full  = get_bool_opt(options, 'full', False)
+        self.full = get_bool_opt(options, 'full', False)
         self.title = options.get('title', '')
         self.encoding = options.get('encoding', None) or None
         if self.encoding in ('guess', 'chardet'):

packages/wakatime/packages/pygments/formatters/__init__.py

@@ -5,7 +5,7 @@
 
     Pygments formatters.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -20,7 +20,7 @@ from pygments.plugin import find_plugin_formatters
 from pygments.util import ClassNotFound, itervalues
 
 __all__ = ['get_formatter_by_name', 'get_formatter_for_filename',
-           'get_all_formatters'] + list(FORMATTERS)
+           'get_all_formatters', 'load_formatter_from_file'] + list(FORMATTERS)
 
 _formatter_cache = {}  # classes by name
 _pattern_cache = {}
@@ -75,10 +75,45 @@ def get_formatter_by_name(_alias, **options):
     """
     cls = find_formatter_class(_alias)
     if cls is None:
-        raise ClassNotFound("No formatter found for name %r" % _alias)
+        raise ClassNotFound("no formatter found for name %r" % _alias)
     return cls(**options)
 
 
+def load_formatter_from_file(filename, formattername="CustomFormatter",
+                             **options):
+    """Load a formatter from a file.
+
+    This method expects a file located relative to the current working
+    directory, which contains a class named CustomFormatter. By default,
+    it expects the Formatter to be named CustomFormatter; you can specify
+    your own class name as the second argument to this function.
+
+    Users should be very careful with the input, because this method
+    is equivalent to running eval on the input file.
+
+    Raises ClassNotFound if there are any problems importing the Formatter.
+
+    .. versionadded:: 2.2
+    """
+    try:
+        # This empty dict will contain the namespace for the exec'd file
+        custom_namespace = {}
+        exec(open(filename, 'rb').read(), custom_namespace)
+        # Retrieve the class `formattername` from that namespace
+        if formattername not in custom_namespace:
+            raise ClassNotFound('no valid %s class found in %s' %
+                                (formattername, filename))
+        formatter_class = custom_namespace[formattername]
+        # And finally instantiate it with the options
+        return formatter_class(**options)
+    except IOError as err:
+        raise ClassNotFound('cannot read %s' % filename)
+    except ClassNotFound as err:
+        raise
+    except Exception as err:
+        raise ClassNotFound('error when loading custom formatter: %s' % err)
+
+
 def get_formatter_for_filename(fn, **options):
     """Lookup and instantiate a formatter by filename pattern.
 
@@ -95,7 +130,7 @@ def get_formatter_for_filename(fn, **options):
         for filename in cls.filenames:
             if _fn_matches(fn, filename):
                 return cls(**options)
-    raise ClassNotFound("No formatter found for file name %r" % fn)
+    raise ClassNotFound("no formatter found for file name %r" % fn)
 
 
 class _automodule(types.ModuleType):

packages/wakatime/packages/pygments/formatters/_mapping.py

@@ -9,7 +9,7 @@
 
     Do not alter the FORMATTERS dictionary by hand.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -20,6 +20,7 @@ FORMATTERS = {
     'BmpImageFormatter': ('pygments.formatters.img', 'img_bmp', ('bmp', 'bitmap'), ('*.bmp',), 'Create a bitmap image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
     'GifImageFormatter': ('pygments.formatters.img', 'img_gif', ('gif',), ('*.gif',), 'Create a GIF image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
     'HtmlFormatter': ('pygments.formatters.html', 'HTML', ('html',), ('*.html', '*.htm'), "Format tokens as HTML 4 ``<span>`` tags within a ``<pre>`` tag, wrapped in a ``<div>`` tag. The ``<div>``'s CSS class can be set by the `cssclass` option."),
+    'IRCFormatter': ('pygments.formatters.irc', 'IRC', ('irc', 'IRC'), (), 'Format tokens with IRC color sequences'),
     'ImageFormatter': ('pygments.formatters.img', 'img', ('img', 'IMG', 'png'), ('*.png',), 'Create a PNG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
     'JpgImageFormatter': ('pygments.formatters.img', 'img_jpg', ('jpg', 'jpeg'), ('*.jpg',), 'Create a JPEG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
     'LatexFormatter': ('pygments.formatters.latex', 'LaTeX', ('latex', 'tex'), ('*.tex',), 'Format tokens as LaTeX code. This needs the `fancyvrb` and `color` standard packages.'),
@@ -27,12 +28,13 @@ FORMATTERS = {
     'RawTokenFormatter': ('pygments.formatters.other', 'Raw tokens', ('raw', 'tokens'), ('*.raw',), 'Format tokens as a raw representation for storing token streams.'),
     'RtfFormatter': ('pygments.formatters.rtf', 'RTF', ('rtf',), ('*.rtf',), 'Format tokens as RTF markup. This formatter automatically outputs full RTF documents with color information and other useful stuff. Perfect for Copy and Paste into Microsoft(R) Word(R) documents.'),
     'SvgFormatter': ('pygments.formatters.svg', 'SVG', ('svg',), ('*.svg',), 'Format tokens as an SVG graphics file.  This formatter is still experimental. Each line of code is a ``<text>`` element with explicit ``x`` and ``y`` coordinates containing ``<tspan>`` elements with the individual token styles.'),
-    'Terminal256Formatter': ('pygments.formatters.terminal256', 'Terminal256', ('terminal256', 'console256', '256'), (), 'Format tokens with ANSI color sequences, for output in a 256-color terminal or console. Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'),
+    'Terminal256Formatter': ('pygments.formatters.terminal256', 'Terminal256', ('terminal256', 'console256', '256'), (), 'Format tokens with ANSI color sequences, for output in a 256-color terminal or console.  Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'),
     'TerminalFormatter': ('pygments.formatters.terminal', 'Terminal', ('terminal', 'console'), (), 'Format tokens with ANSI color sequences, for output in a text console. Color sequences are terminated at newlines, so that paging the output works correctly.'),
+    'TerminalTrueColorFormatter': ('pygments.formatters.terminal256', 'TerminalTrueColor', ('terminal16m', 'console16m', '16m'), (), 'Format tokens with ANSI color sequences, for output in a true-color terminal or console.  Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'),
     'TestcaseFormatter': ('pygments.formatters.other', 'Testcase', ('testcase',), (), 'Format tokens as appropriate for a new testcase.')
 }
 
-if __name__ == '__main__':
+if __name__ == '__main__':  # pragma: no cover
     import sys
     import os
 
@@ -64,6 +66,13 @@ if __name__ == '__main__':
     # extract useful sourcecode from this file
     with open(__file__) as fp:
         content = fp.read()
+        # replace crnl to nl for Windows.
+        #
+        # Note that, originally, contributers should keep nl of master
+        # repository, for example by using some kind of automatic
+        # management EOL, like `EolExtension
+        #  <https://www.mercurial-scm.org/wiki/EolExtension>`.
+        content = content.replace("\r\n", "\n")
     header = content[:content.find('FORMATTERS = {')]
     footer = content[content.find("if __name__ == '__main__':"):]
 

packages/wakatime/packages/pygments/formatters/bbcode.py

@@ -5,7 +5,7 @@
 
     BBcode formatter.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/formatters/html.py

@@ -5,7 +5,7 @@
 
     Formatter for HTML output.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -36,21 +36,11 @@ _escape_html_table = {
     ord("'"): u''',
 }
 
+
 def escape_html(text, table=_escape_html_table):
     """Escape &, <, > as well as single and double quotes for HTML."""
     return text.translate(table)
 
-def get_random_id():
-    """Return a random id for javascript fields."""
-    from random import random
-    from time import time
-    try:
-        from hashlib import sha1 as sha
-    except ImportError:
-        import sha
-        sha = sha.new
-    return sha('%s|%s' % (random(), time())).hexdigest()
-
 
 def _get_ttype_class(ttype):
     fname = STANDARD_TYPES.get(ttype)
@@ -150,7 +140,7 @@ class HtmlFormatter(Formatter):
 
     When `tagsfile` is set to the path of a ctags index file, it is used to
     generate hyperlinks from names to their definition.  You must enable
-    `anchorlines` and run ctags with the `-n` option for this to work.  The
+    `lineanchors` and run ctags with the `-n` option for this to work.  The
     `python-ctags` module from PyPI must be installed to use this feature;
     otherwise a `RuntimeError` will be raised.
 
@@ -331,6 +321,12 @@ class HtmlFormatter(Formatter):
 
         .. versionadded:: 1.6
 
+    `filename`
+        A string used to generate a filename when rendering <pre> blocks,
+        for example if displaying source code.
+
+        .. versionadded:: 2.1
+
 
     **Subclassing the HTML formatter**
 
@@ -398,6 +394,7 @@ class HtmlFormatter(Formatter):
         self.noclobber_cssfile = get_bool_opt(options, 'noclobber_cssfile', False)
         self.tagsfile = self._decodeifneeded(options.get('tagsfile', ''))
         self.tagurlformat = self._decodeifneeded(options.get('tagurlformat', ''))
+        self.filename = self._decodeifneeded(options.get('filename', ''))
 
         if self.tagsfile:
             if not ctags:
@@ -438,6 +435,15 @@ class HtmlFormatter(Formatter):
             return self.classprefix + ttypeclass
         return ''
 
+    def _get_css_classes(self, ttype):
+        """Return the css classes of this token type prefixed with
+        the classprefix option."""
+        cls = self._get_css_class(ttype)
+        while ttype not in STANDARD_TYPES:
+            ttype = ttype.parent
+            cls = self._get_css_class(ttype) + ' ' + cls
+        return cls
+
     def _create_stylesheet(self):
         t2c = self.ttype2class = {Token: ''}
         c2s = self.class2style = {}
@@ -522,7 +528,7 @@ class HtmlFormatter(Formatter):
                     cssfilename = os.path.join(os.path.dirname(filename),
                                                self.cssfile)
                 except AttributeError:
-                    print('Note: Cannot determine output file name, ' \
+                    print('Note: Cannot determine output file name, '
                           'using current directory as base for the CSS file name',
                           file=sys.stderr)
                     cssfilename = self.cssfile
@@ -531,21 +537,21 @@ class HtmlFormatter(Formatter):
                 if not os.path.exists(cssfilename) or not self.noclobber_cssfile:
                     cf = open(cssfilename, "w")
                     cf.write(CSSFILE_TEMPLATE %
-                            {'styledefs': self.get_style_defs('body')})
+                             {'styledefs': self.get_style_defs('body')})
                     cf.close()
             except IOError as err:
                 err.strerror = 'Error writing CSS file: ' + err.strerror
                 raise
 
             yield 0, (DOC_HEADER_EXTERNALCSS %
-                      dict(title     = self.title,
-                           cssfile   = self.cssfile,
-                           encoding  = self.encoding))
+                      dict(title=self.title,
+                           cssfile=self.cssfile,
+                           encoding=self.encoding))
         else:
             yield 0, (DOC_HEADER %
-                      dict(title     = self.title,
-                           styledefs = self.get_style_defs('body'),
-                           encoding  = self.encoding))
+                      dict(title=self.title,
+                           styledefs=self.get_style_defs('body'),
+                           encoding=self.encoding))
 
         for t, line in inner:
             yield t, line
@@ -624,35 +630,35 @@ class HtmlFormatter(Formatter):
         if self.noclasses:
             if sp:
                 for t, line in lines:
-                    if num%sp == 0:
+                    if num % sp == 0:
                         style = 'background-color: #ffffc0; padding: 0 5px 0 5px'
                     else:
                         style = 'background-color: #f0f0f0; padding: 0 5px 0 5px'
                     yield 1, '<span style="%s">%*s </span>' % (
-                        style, mw, (num%st and ' ' or num)) + line
+                        style, mw, (num % st and ' ' or num)) + line
                     num += 1
             else:
                 for t, line in lines:
                     yield 1, ('<span style="background-color: #f0f0f0; '
                               'padding: 0 5px 0 5px">%*s </span>' % (
-                              mw, (num%st and ' ' or num)) + line)
+                                  mw, (num % st and ' ' or num)) + line)
                     num += 1
         elif sp:
             for t, line in lines:
                 yield 1, '<span class="lineno%s">%*s </span>' % (
-                    num%sp == 0 and ' special' or '', mw,
-                    (num%st and ' ' or num)) + line
+                    num % sp == 0 and ' special' or '', mw,
+                    (num % st and ' ' or num)) + line
                 num += 1
         else:
             for t, line in lines:
                 yield 1, '<span class="lineno">%*s </span>' % (
-                    mw, (num%st and ' ' or num)) + line
+                    mw, (num % st and ' ' or num)) + line
                 num += 1
 
     def _wrap_lineanchors(self, inner):
         s = self.lineanchors
-        i = self.linenostart - 1 # subtract 1 since we have to increment i
-                                 # *before* yielding
+        # subtract 1 since we have to increment i *before* yielding
+        i = self.linenostart - 1
         for t, line in inner:
             if t:
                 i += 1
@@ -673,14 +679,14 @@ class HtmlFormatter(Formatter):
     def _wrap_div(self, inner):
         style = []
         if (self.noclasses and not self.nobackground and
-            self.style.background_color is not None):
+                self.style.background_color is not None):
             style.append('background: %s' % (self.style.background_color,))
         if self.cssstyles:
             style.append(self.cssstyles)
         style = '; '.join(style)
 
-        yield 0, ('<div' + (self.cssclass and ' class="%s"' % self.cssclass)
-                  + (style and (' style="%s"' % style)) + '>')
+        yield 0, ('<div' + (self.cssclass and ' class="%s"' % self.cssclass) +
+                  (style and (' style="%s"' % style)) + '>')
         for tup in inner:
             yield tup
         yield 0, '</div>\n'
@@ -693,7 +699,12 @@ class HtmlFormatter(Formatter):
             style.append('line-height: 125%')
         style = '; '.join(style)
 
-        yield 0, ('<pre' + (style and ' style="%s"' % style) + '>')
+        if self.filename:
+            yield 0, ('<span class="filename">' + self.filename + '</span>')
+
+        # the empty span here is to keep leading empty lines from being
+        # ignored by HTML parsers
+        yield 0, ('<pre' + (style and ' style="%s"' % style) + '><span></span>')
         for tup in inner:
             yield tup
         yield 0, '</pre>'
@@ -712,7 +723,7 @@ class HtmlFormatter(Formatter):
         tagsfile = self.tagsfile
 
         lspan = ''
-        line = ''
+        line = []
         for ttype, value in tokensource:
             if nocls:
                 cclass = getcls(ttype)
@@ -721,7 +732,7 @@ class HtmlFormatter(Formatter):
                     cclass = getcls(ttype)
                 cspan = cclass and '<span style="%s">' % c2s[cclass][0] or ''
             else:
-                cls = self._get_css_class(ttype)
+                cls = self._get_css_classes(ttype)
                 cspan = cls and '<span class="%s">' % cls or ''
 
             parts = value.translate(escape_table).split('\n')
@@ -743,30 +754,31 @@ class HtmlFormatter(Formatter):
             for part in parts[:-1]:
                 if line:
                     if lspan != cspan:
-                        line += (lspan and '</span>') + cspan + part + \
-                                (cspan and '</span>') + lsep
-                    else: # both are the same
-                        line += part + (lspan and '</span>') + lsep
-                    yield 1, line
-                    line = ''
+                        line.extend(((lspan and '</span>'), cspan, part,
+                                     (cspan and '</span>'), lsep))
+                    else:  # both are the same
+                        line.extend((part, (lspan and '</span>'), lsep))
+                    yield 1, ''.join(line)
+                    line = []
                 elif part:
-                    yield 1, cspan + part + (cspan and '</span>') + lsep
+                    yield 1, ''.join((cspan, part, (cspan and '</span>'), lsep))
                 else:
                     yield 1, lsep
             # for the last line
             if line and parts[-1]:
                 if lspan != cspan:
-                    line += (lspan and '</span>') + cspan + parts[-1]
+                    line.extend(((lspan and '</span>'), cspan, parts[-1]))
                     lspan = cspan
                 else:
-                    line += parts[-1]
+                    line.append(parts[-1])
             elif parts[-1]:
-                line = cspan + parts[-1]
+                line = [cspan, parts[-1]]
                 lspan = cspan
             # else we neither have to open a new span nor set lspan
 
         if line:
-            yield 1, line + (lspan and '</span>') + lsep
+            line.extend(((lspan and '</span>'), lsep))
+            yield 1, ''.join(line)
 
     def _lookup_ctag(self, token):
         entry = ctags.TagEntry()
@@ -785,7 +797,7 @@ class HtmlFormatter(Formatter):
         for i, (t, value) in enumerate(tokensource):
             if t != 1:
                 yield t, value
-            if i + 1 in hls: # i + 1 because Python indexes start at 0
+            if i + 1 in hls:  # i + 1 because Python indexes start at 0
                 if self.noclasses:
                     style = ''
                     if self.style.highlight_color is not None:

packages/wakatime/packages/pygments/formatters/img.py

@@ -5,16 +5,19 @@
 
     Formatter for Pixmap output.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+import os
 import sys
 
 from pygments.formatter import Formatter
 from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
     get_choice_opt, xrange
 
+import subprocess
+
 # Import this carefully
 try:
     from PIL import Image, ImageDraw, ImageFont
@@ -45,6 +48,7 @@ STYLES = {
 # A sane default for modern systems
 DEFAULT_FONT_NAME_NIX = 'Bitstream Vera Sans Mono'
 DEFAULT_FONT_NAME_WIN = 'Courier New'
+DEFAULT_FONT_NAME_MAC = 'Courier New'
 
 
 class PilNotAvailable(ImportError):
@@ -69,23 +73,28 @@ class FontManager(object):
             if not font_name:
                 self.font_name = DEFAULT_FONT_NAME_WIN
             self._create_win()
+        elif sys.platform.startswith('darwin'):
+            if not font_name:
+                self.font_name = DEFAULT_FONT_NAME_MAC
+            self._create_mac()
         else:
             if not font_name:
                 self.font_name = DEFAULT_FONT_NAME_NIX
             self._create_nix()
 
     def _get_nix_font_path(self, name, style):
-        try:
-            from commands import getstatusoutput
-        except ImportError:
-            from subprocess import getstatusoutput
-        exit, out = getstatusoutput('fc-list "%s:style=%s" file' %
-                                    (name, style))
-        if not exit:
-            lines = out.splitlines()
-            if lines:
-                path = lines[0].strip().strip(':')
-                return path
+        proc = subprocess.Popen(['fc-list', "%s:style=%s" % (name, style), 'file'],
+                                stdout=subprocess.PIPE, stderr=None)
+        stdout, _ = proc.communicate()
+        if proc.returncode == 0:
+            lines = stdout.splitlines()
+            for line in lines:
+                if line.startswith(b'Fontconfig warning:'):
+                    continue
+                path = line.decode().strip().strip(':')
+                if path:
+                    return path
+            return None
 
     def _create_nix(self):
         for name in STYLES['NORMAL']:
@@ -108,6 +117,37 @@ class FontManager(object):
                 else:
                     self.fonts[style] = self.fonts['NORMAL']
 
+    def _get_mac_font_path(self, font_map, name, style):
+        return font_map.get((name + ' ' + style).strip().lower())
+
+    def _create_mac(self):
+        font_map = {}
+        for font_dir in (os.path.join(os.getenv("HOME"), 'Library/Fonts/'),
+                         '/Library/Fonts/', '/System/Library/Fonts/'):
+            font_map.update(
+                ((os.path.splitext(f)[0].lower(), os.path.join(font_dir, f))
+                    for f in os.listdir(font_dir) if f.lower().endswith('ttf')))
+
+        for name in STYLES['NORMAL']:
+            path = self._get_mac_font_path(font_map, self.font_name, name)
+            if path is not None:
+                self.fonts['NORMAL'] = ImageFont.truetype(path, self.font_size)
+                break
+        else:
+            raise FontNotFound('No usable fonts named: "%s"' %
+                               self.font_name)
+        for style in ('ITALIC', 'BOLD', 'BOLDITALIC'):
+            for stylename in STYLES[style]:
+                path = self._get_mac_font_path(font_map, self.font_name, stylename)
+                if path is not None:
+                    self.fonts[style] = ImageFont.truetype(path, self.font_size)
+                    break
+            else:
+                if style == 'BOLDITALIC':
+                    self.fonts[style] = self.fonts['BOLD']
+                else:
+                    self.fonts[style] = self.fonts['NORMAL']
+
     def _lookup_win(self, key, basename, styles, fail=False):
         for suffix in ('', ' (TrueType)'):
             for style in styles:
@@ -197,7 +237,7 @@ class ImageFormatter(Formatter):
         bold and italic fonts will be generated.  This really should be a
         monospace font to look sane.
 
-        Default: "Bitstream Vera Sans Mono"
+        Default: "Bitstream Vera Sans Mono" on Windows, Courier New on \*nix
 
     `font_size`
         The font size in points to be used.

packages/wakatime/packages/pygments/formatters/irc.py

@@ -1,11 +1,11 @@
 # -*- coding: utf-8 -*-
 """
-    pygments.formatters.terminal
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    pygments.formatters.irc
+    ~~~~~~~~~~~~~~~~~~~~~~~
 
-    Formatter for terminal output with ANSI sequences.
+    Formatter for IRC output
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -14,16 +14,15 @@ import sys
 from pygments.formatter import Formatter
 from pygments.token import Keyword, Name, Comment, String, Error, \
     Number, Operator, Generic, Token, Whitespace
-from pygments.console import ansiformat
 from pygments.util import get_choice_opt
 
 
-__all__ = ['TerminalFormatter']
+__all__ = ['IRCFormatter']
 
 
 #: Map token types to a tuple of color values for light and dark
 #: backgrounds.
-TERMINAL_COLORS = {
+IRC_COLORS = {
     Token:              ('',            ''),
 
     Whitespace:         ('lightgray',   'darkgray'),
@@ -55,11 +54,52 @@ TERMINAL_COLORS = {
 }
 
 
-class TerminalFormatter(Formatter):
+IRC_COLOR_MAP = {
+    'white': 0,
+    'black': 1,
+    'darkblue': 2,
+    'green': 3,
+    'red': 4,
+    'brown': 5,
+    'purple': 6,
+    'orange': 7,
+    'darkgreen': 7, #compat w/ ansi
+    'yellow': 8,
+    'lightgreen': 9,
+    'turquoise': 9, # compat w/ ansi
+    'teal': 10,
+    'lightblue': 11,
+    'darkred': 11, # compat w/ ansi
+    'blue': 12,
+    'fuchsia': 13,
+    'darkgray': 14,
+    'lightgray': 15,
+}
+
+def ircformat(color, text):
+    if len(color) < 1:
+        return text
+    add = sub = ''
+    if '_' in color: # italic
+        add += '\x1D'
+        sub = '\x1D' + sub
+        color = color.strip('_')
+    if '*' in color: # bold
+        add += '\x02'
+        sub = '\x02' + sub
+        color = color.strip('*')
+    # underline (\x1F) not supported
+    # backgrounds (\x03FF,BB) not supported
+    if len(color) > 0: # actual color - may have issues with ircformat("red", "blah")+"10" type stuff
+        add += '\x03' + str(IRC_COLOR_MAP[color]).zfill(2)
+        sub = '\x03' + sub
+    return add + text + sub
+    return '<'+add+'>'+text+'</'+sub+'>'
+
+
+class IRCFormatter(Formatter):
     r"""
-    Format tokens with ANSI color sequences, for output in a text console.
-    Color sequences are terminated at newlines, so that paging the output
-    works correctly.
+    Format tokens with IRC color sequences
 
     The `get_style_defs()` method doesn't do anything special since there is
     no support for common styles.
@@ -75,30 +115,21 @@ class TerminalFormatter(Formatter):
         ``None`` (default: ``None`` = use builtin colorscheme).
 
     `linenos`
-        Set to ``True`` to have line numbers on the terminal output as well
+        Set to ``True`` to have line numbers in the output as well
         (default: ``False`` = no line numbers).
     """
-    name = 'Terminal'
-    aliases = ['terminal', 'console']
+    name = 'IRC'
+    aliases = ['irc', 'IRC']
     filenames = []
 
     def __init__(self, **options):
         Formatter.__init__(self, **options)
         self.darkbg = get_choice_opt(options, 'bg',
                                      ['light', 'dark'], 'light') == 'dark'
-        self.colorscheme = options.get('colorscheme', None) or TERMINAL_COLORS
+        self.colorscheme = options.get('colorscheme', None) or IRC_COLORS
         self.linenos = options.get('linenos', False)
         self._lineno = 0
 
-    def format(self, tokensource, outfile):
-        # hack: if the output is a terminal and has an encoding set,
-        # use that to avoid unicode encode problems
-        if not self.encoding and hasattr(outfile, "encoding") and \
-           hasattr(outfile, "isatty") and outfile.isatty() and \
-           sys.version_info < (3,):
-            self.encoding = outfile.encoding
-        return Formatter.format(self, tokensource, outfile)
-
     def _write_lineno(self, outfile):
         self._lineno += 1
         outfile.write("\n%04d: " % self._lineno)
@@ -120,9 +151,9 @@ class TerminalFormatter(Formatter):
                 for line in spl[:-1]:
                     self._write_lineno(outfile)
                     if line:
-                        outfile.write(ansiformat(color, line[:-1]))
+                        outfile.write(ircformat(color, line[:-1]))
                 if spl[-1]:
-                    outfile.write(ansiformat(color, spl[-1]))
+                    outfile.write(ircformat(color, spl[-1]))
             else:
                 outfile.write(value)
 
@@ -143,9 +174,9 @@ class TerminalFormatter(Formatter):
                 spl = value.split('\n')
                 for line in spl[:-1]:
                     if line:
-                        outfile.write(ansiformat(color, line))
+                        outfile.write(ircformat(color, line))
                     outfile.write('\n')
                 if spl[-1]:
-                    outfile.write(ansiformat(color, spl[-1]))
+                    outfile.write(ircformat(color, spl[-1]))
             else:
                 outfile.write(value)

packages/wakatime/packages/pygments/formatters/latex.py

@@ -5,7 +5,7 @@
 
     Formatter for LaTeX fancyvrb output.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -360,7 +360,7 @@ class LatexFormatter(Formatter):
                         start += value[i]
 
                     value = value[len(start):]
-                    start = escape_tex(start, self.commandprefix)
+                    start = escape_tex(start, cp)
 
                     # ... but do not escape inside comment.
                     value = start + value
@@ -370,26 +370,26 @@ class LatexFormatter(Formatter):
                     in_math = False
                     for i, part in enumerate(parts):
                         if not in_math:
-                            parts[i] = escape_tex(part, self.commandprefix)
+                            parts[i] = escape_tex(part, cp)
                         in_math = not in_math
                     value = '$'.join(parts)
                 elif self.escapeinside:
                     text = value
                     value = ''
-                    while len(text) > 0:
+                    while text:
                         a, sep1, text = text.partition(self.left)
-                        if len(sep1) > 0:
+                        if sep1:
                             b, sep2, text = text.partition(self.right)
-                            if len(sep2) > 0:
-                                value += escape_tex(a, self.commandprefix) + b
+                            if sep2:
+                                value += escape_tex(a, cp) + b
                             else:
-                                value += escape_tex(a + sep1 + b, self.commandprefix)
+                                value += escape_tex(a + sep1 + b, cp)
                         else:
-                            value = value + escape_tex(a, self.commandprefix)
+                            value += escape_tex(a, cp)
                 else:
-                    value = escape_tex(value, self.commandprefix)
+                    value = escape_tex(value, cp)
             elif ttype not in Token.Escape:
-                value = escape_tex(value, self.commandprefix)
+                value = escape_tex(value, cp)
             styles = []
             while ttype is not Token:
                 try:
@@ -413,18 +413,24 @@ class LatexFormatter(Formatter):
         outfile.write(u'\\end{' + self.envname + u'}\n')
 
         if self.full:
+            encoding = self.encoding or 'utf8'
+            # map known existings encodings from LaTeX distribution
+            encoding = {
+                'utf_8': 'utf8',
+                'latin_1': 'latin1',
+                'iso_8859_1': 'latin1',
+            }.get(encoding.replace('-', '_'), encoding)
             realoutfile.write(DOC_TEMPLATE %
                 dict(docclass  = self.docclass,
                      preamble  = self.preamble,
                      title     = self.title,
-                     encoding  = self.encoding or 'utf8',
+                     encoding  = encoding,
                      styledefs = self.get_style_defs(),
                      code      = outfile.getvalue()))
 
 
 class LatexEmbeddedLexer(Lexer):
-    r"""
-
+    """
     This lexer takes one lexer as argument, the lexer for the language
     being formatted, and the left and right delimiters for escaped text.
 

packages/wakatime/packages/pygments/formatters/other.py

@@ -5,7 +5,7 @@
 
     Other formatters: NullFormatter, RawTokenFormatter.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/formatters/rtf.py

@@ -5,7 +5,7 @@
 
     A formatter that generates RTF files.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/formatters/svg.py

@@ -5,7 +5,7 @@
 
     Formatter for SVG output.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/formatters/terminal.py

@@ -5,7 +5,7 @@
 
     Formatter for terminal output with ANSI sequences.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -49,6 +49,7 @@ TERMINAL_COLORS = {
     Generic.Inserted:   ('darkgreen',  'green'),
     Generic.Heading:    ('**',         '**'),
     Generic.Subheading: ('*purple*',   '*fuchsia*'),
+    Generic.Prompt:     ('**',         '**'),
     Generic.Error:      ('red',        'red'),
 
     Error:              ('_red_',      '_red_'),
@@ -101,51 +102,35 @@ class TerminalFormatter(Formatter):
 
     def _write_lineno(self, outfile):
         self._lineno += 1
-        outfile.write("\n%04d: " % self._lineno)
+        outfile.write("%s%04d: " % (self._lineno != 1 and '\n' or '', self._lineno))
 
-    def _format_unencoded_with_lineno(self, tokensource, outfile):
-        self._write_lineno(outfile)
-
-        for ttype, value in tokensource:
-            if value.endswith("\n"):
-                self._write_lineno(outfile)
-                value = value[:-1]
-            color = self.colorscheme.get(ttype)
-            while color is None:
-                ttype = ttype[:-1]
-                color = self.colorscheme.get(ttype)
-            if color:
-                color = color[self.darkbg]
-                spl = value.split('\n')
-                for line in spl[:-1]:
-                    self._write_lineno(outfile)
-                    if line:
-                        outfile.write(ansiformat(color, line[:-1]))
-                if spl[-1]:
-                    outfile.write(ansiformat(color, spl[-1]))
-            else:
-                outfile.write(value)
-
-        outfile.write("\n")
+    def _get_color(self, ttype):
+        # self.colorscheme is a dict containing usually generic types, so we
+        # have to walk the tree of dots.  The base Token type must be a key,
+        # even if it's empty string, as in the default above.
+        colors = self.colorscheme.get(ttype)
+        while colors is None:
+            ttype = ttype.parent
+            colors = self.colorscheme.get(ttype)
+        return colors[self.darkbg]
 
     def format_unencoded(self, tokensource, outfile):
         if self.linenos:
-            self._format_unencoded_with_lineno(tokensource, outfile)
-            return
+            self._write_lineno(outfile)
 
         for ttype, value in tokensource:
-            color = self.colorscheme.get(ttype)
-            while color is None:
-                ttype = ttype[:-1]
-                color = self.colorscheme.get(ttype)
-            if color:
-                color = color[self.darkbg]
-                spl = value.split('\n')
-                for line in spl[:-1]:
-                    if line:
-                        outfile.write(ansiformat(color, line))
-                    outfile.write('\n')
-                if spl[-1]:
-                    outfile.write(ansiformat(color, spl[-1]))
-            else:
-                outfile.write(value)
+            color = self._get_color(ttype)
+
+            for line in value.splitlines(True):
+                if color:
+                    outfile.write(ansiformat(color, line.rstrip('\n')))
+                else:
+                    outfile.write(line.rstrip('\n'))
+                if line.endswith('\n'):
+                    if self.linenos:
+                        self._write_lineno(outfile)
+                    else:
+                        outfile.write('\n')
+
+        if self.linenos:
+            outfile.write("\n")

packages/wakatime/packages/pygments/formatters/terminal256.py

@@ -11,7 +11,7 @@
 
     Formatter version 1.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -27,9 +27,11 @@
 import sys
 
 from pygments.formatter import Formatter
+from pygments.console import codes
+from pygments.style import ansicolors
 
 
-__all__ = ['Terminal256Formatter']
+__all__ = ['Terminal256Formatter', 'TerminalTrueColorFormatter']
 
 
 class EscapeSequence:
@@ -47,9 +49,33 @@ class EscapeSequence:
     def color_string(self):
         attrs = []
         if self.fg is not None:
-            attrs.extend(("38", "5", "%i" % self.fg))
+            if self.fg in ansicolors:
+                esc = codes[self.fg[5:]]
+                if ';01m' in esc:
+                    self.bold = True
+                # extract fg color code.
+                attrs.append(esc[2:4])
+            else:
+                attrs.extend(("38", "5", "%i" % self.fg))
         if self.bg is not None:
-            attrs.extend(("48", "5", "%i" % self.bg))
+            if self.bg in ansicolors:
+                esc = codes[self.bg[5:]]
+                # extract fg color code, add 10 for bg.
+                attrs.append(str(int(esc[2:4])+10))
+            else:
+                attrs.extend(("48", "5", "%i" % self.bg))
+        if self.bold:
+            attrs.append("01")
+        if self.underline:
+            attrs.append("04")
+        return self.escape(attrs)
+
+    def true_color_string(self):
+        attrs = []
+        if self.fg:
+            attrs.extend(("38", "2", str(self.fg[0]), str(self.fg[1]), str(self.fg[2])))
+        if self.bg:
+            attrs.extend(("48", "2", str(self.bg[0]), str(self.bg[1]), str(self.bg[2])))
         if self.bold:
             attrs.append("01")
         if self.underline:
@@ -68,9 +94,9 @@ class EscapeSequence:
 
 
 class Terminal256Formatter(Formatter):
-    r"""
+    """
     Format tokens with ANSI color sequences, for output in a 256-color
-    terminal or console. Like in `TerminalFormatter` color sequences
+    terminal or console.  Like in `TerminalFormatter` color sequences
     are terminated at newlines, so that paging the output works correctly.
 
     The formatter takes colors from a style defined by the `style` option
@@ -79,6 +105,11 @@ class Terminal256Formatter(Formatter):
 
     .. versionadded:: 0.9
 
+    .. versionchanged:: 2.2
+       If the used style defines foreground colors in the form ``#ansi*``, then
+       `Terminal256Formatter` will map these to non extended foreground color.
+       See :ref:`AnsiTerminalStyle` for more information.
+
     Options accepted:
 
     `style`
@@ -157,6 +188,10 @@ class Terminal256Formatter(Formatter):
 
     def _color_index(self, color):
         index = self.best_match.get(color, None)
+        if color in ansicolors:
+            # strip the `#ansi` part and look up code
+            index = color
+            self.best_match[color] = index
         if index is None:
             try:
                 rgb = int(str(color), 16)
@@ -173,9 +208,14 @@ class Terminal256Formatter(Formatter):
     def _setup_styles(self):
         for ttype, ndef in self.style:
             escape = EscapeSequence()
-            if ndef['color']:
+            # get foreground from ansicolor if set
+            if ndef['ansicolor']:
+                escape.fg = self._color_index(ndef['ansicolor'])
+            elif ndef['color']:
                 escape.fg = self._color_index(ndef['color'])
-            if ndef['bgcolor']:
+            if ndef['bgansicolor']:
+                escape.bg = self._color_index(ndef['bgansicolor'])
+            elif ndef['bgcolor']:
                 escape.bg = self._color_index(ndef['bgcolor'])
             if self.usebold and ndef['bold']:
                 escape.bold = True
@@ -221,3 +261,49 @@ class Terminal256Formatter(Formatter):
 
             if not_found:
                 outfile.write(value)
+
+
+class TerminalTrueColorFormatter(Terminal256Formatter):
+    r"""
+    Format tokens with ANSI color sequences, for output in a true-color
+    terminal or console.  Like in `TerminalFormatter` color sequences
+    are terminated at newlines, so that paging the output works correctly.
+
+    .. versionadded:: 2.1
+
+    Options accepted:
+
+    `style`
+        The style to use, can be a string or a Style subclass (default:
+        ``'default'``).
+    """
+    name = 'TerminalTrueColor'
+    aliases = ['terminal16m', 'console16m', '16m']
+    filenames = []
+
+    def _build_color_table(self):
+        pass
+
+    def _color_tuple(self, color):
+        try:
+            rgb = int(str(color), 16)
+        except ValueError:
+            return None
+        r = (rgb >> 16) & 0xff
+        g = (rgb >> 8) & 0xff
+        b = rgb & 0xff
+        return (r, g, b)
+
+    def _setup_styles(self):
+        for ttype, ndef in self.style:
+            escape = EscapeSequence()
+            if ndef['color']:
+                escape.fg = self._color_tuple(ndef['color'])
+            if ndef['bgcolor']:
+                escape.bg = self._color_tuple(ndef['bgcolor'])
+            if self.usebold and ndef['bold']:
+                escape.bold = True
+            if self.useunderline and ndef['underline']:
+                escape.underline = True
+            self.style_string[str(ttype)] = (escape.true_color_string(),
+                                             escape.reset_string())

packages/wakatime/packages/pygments/lexer.py

@@ -5,7 +5,7 @@
 
     Base lexer classes.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -14,7 +14,6 @@ from __future__ import print_function
 import re
 import sys
 import time
-import itertools
 
 from pygments.filter import apply_filters, Filter
 from pygments.filters import get_filter_by_name
@@ -43,10 +42,10 @@ class LexerMeta(type):
     static methods which always return float values.
     """
 
-    def __new__(cls, name, bases, d):
+    def __new__(mcs, name, bases, d):
         if 'analyse_text' in d:
             d['analyse_text'] = make_analysator(d['analyse_text'])
-        return type.__new__(cls, name, bases, d)
+        return type.__new__(mcs, name, bases, d)
 
 
 @add_metaclass(LexerMeta)
@@ -189,7 +188,7 @@ class Lexer(object):
             text += '\n'
 
         def streamer():
-            for i, t, v in self.get_tokens_unprocessed(text):
+            for _, t, v in self.get_tokens_unprocessed(text):
                 yield t, v
         stream = streamer()
         if not unfiltered:
@@ -246,7 +245,7 @@ class DelegatingLexer(Lexer):
 #
 
 
-class include(str):
+class include(str):  # pylint: disable=invalid-name
     """
     Indicates that a state should include rules from another state.
     """
@@ -260,10 +259,10 @@ class _inherit(object):
     def __repr__(self):
         return 'inherit'
 
-inherit = _inherit()
+inherit = _inherit()  # pylint: disable=invalid-name
 
 
-class combined(tuple):
+class combined(tuple):  # pylint: disable=invalid-name
     """
     Indicates a state combined from multiple states.
     """
@@ -320,8 +319,8 @@ def bygroups(*args):
                 if data is not None:
                     if ctx:
                         ctx.pos = match.start(i + 1)
-                    for item in action(lexer, _PseudoMatch(match.start(i + 1),
-                                       data), ctx):
+                    for item in action(lexer,
+                                       _PseudoMatch(match.start(i + 1), data), ctx):
                         if item:
                             yield item
         if ctx:
@@ -655,6 +654,8 @@ class RegexLexer(Lexer):
                         statetokens = tokendefs[statestack[-1]]
                     break
             else:
+                # We are here only if all state tokens have been considered
+                # and there was not a match on any of them.
                 try:
                     if text[pos] == '\n':
                         # at EOL, reset state to "root"

packages/wakatime/packages/pygments/lexers/__init__.py

@@ -5,7 +5,7 @@
 
     Pygments lexers.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -22,7 +22,7 @@ from pygments.util import ClassNotFound, itervalues, guess_decode
 
 
 __all__ = ['get_lexer_by_name', 'get_lexer_for_filename', 'find_lexer_class',
-           'guess_lexer'] + list(LEXERS)
+           'guess_lexer', 'load_lexer_from_file'] + list(LEXERS)
 
 _lexer_cache = {}
 _pattern_cache = {}
@@ -72,6 +72,28 @@ def find_lexer_class(name):
             return cls
 
 
+def find_lexer_class_by_name(_alias):
+    """Lookup a lexer class by alias.
+
+    Like `get_lexer_by_name`, but does not instantiate the class.
+
+    .. versionadded:: 2.2
+    """
+    if not _alias:
+        raise ClassNotFound('no lexer for alias %r found' % _alias)
+    # lookup builtin lexers
+    for module_name, name, aliases, _, _ in itervalues(LEXERS):
+        if _alias.lower() in aliases:
+            if name not in _lexer_cache:
+                _load_lexers(module_name)
+            return _lexer_cache[name]
+    # continue with lexers from setuptools entrypoints
+    for cls in find_plugin_lexers():
+        if _alias.lower() in cls.aliases:
+            return cls
+    raise ClassNotFound('no lexer for alias %r found' % _alias)
+
+
 def get_lexer_by_name(_alias, **options):
     """Get a lexer by an alias.
 
@@ -88,11 +110,45 @@ def get_lexer_by_name(_alias, **options):
             return _lexer_cache[name](**options)
     # continue with lexers from setuptools entrypoints
     for cls in find_plugin_lexers():
-        if _alias in cls.aliases:
+        if _alias.lower() in cls.aliases:
             return cls(**options)
     raise ClassNotFound('no lexer for alias %r found' % _alias)
 
 
+def load_lexer_from_file(filename, lexername="CustomLexer", **options):
+    """Load a lexer from a file.
+
+    This method expects a file located relative to the current working
+    directory, which contains a Lexer class. By default, it expects the
+    Lexer to be name CustomLexer; you can specify your own class name
+    as the second argument to this function.
+
+    Users should be very careful with the input, because this method
+    is equivalent to running eval on the input file.
+
+    Raises ClassNotFound if there are any problems importing the Lexer.
+
+    .. versionadded:: 2.2
+    """
+    try:
+        # This empty dict will contain the namespace for the exec'd file
+        custom_namespace = {}
+        exec(open(filename, 'rb').read(), custom_namespace)
+        # Retrieve the class `lexername` from that namespace
+        if lexername not in custom_namespace:
+            raise ClassNotFound('no valid %s class found in %s' %
+                                (lexername, filename))
+        lexer_class = custom_namespace[lexername]
+        # And finally instantiate it with the options
+        return lexer_class(**options)
+    except IOError as err:
+        raise ClassNotFound('cannot read %s' % filename)
+    except ClassNotFound as err:
+        raise
+    except Exception as err:
+        raise ClassNotFound('error when loading custom lexer: %s' % err)
+
+
 def find_lexer_class_for_filename(_fn, code=None):
     """Get a lexer for a filename.
 
@@ -127,8 +183,8 @@ def find_lexer_class_for_filename(_fn, code=None):
         # gets turned into 0.0.  Run scripts/detect_missing_analyse_text.py
         # to find lexers which need it overridden.
         if code:
-            return cls.analyse_text(code) + bonus
-        return cls.priority + bonus
+            return cls.analyse_text(code) + bonus, cls.__name__
+        return cls.priority + bonus, cls.__name__
 
     if matches:
         matches.sort(key=get_rating)

packages/wakatime/packages/pygments/lexers/_asy_builtins.py

@@ -10,7 +10,7 @@
     TODO: perl/python script in Asymptote SVN similar to asy-list.pl but only
     for function and variable names.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/_cl_builtins.py

@@ -5,7 +5,7 @@
 
     ANSI Common Lisp builtins.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/_lua_builtins.py

@@ -9,60 +9,71 @@
 
     Do not edit the MODULES dict by hand.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 from __future__ import print_function
 
-
 MODULES = {'basic': ('_G',
            '_VERSION',
            'assert',
            'collectgarbage',
            'dofile',
            'error',
-           'getfenv',
            'getmetatable',
            'ipairs',
            'load',
            'loadfile',
-           'loadstring',
            'next',
            'pairs',
            'pcall',
            'print',
            'rawequal',
            'rawget',
+           'rawlen',
            'rawset',
            'select',
-           'setfenv',
            'setmetatable',
            'tonumber',
            'tostring',
            'type',
-           'unpack',
            'xpcall'),
+ 'bit32': ('bit32.arshift',
+           'bit32.band',
+           'bit32.bnot',
+           'bit32.bor',
+           'bit32.btest',
+           'bit32.bxor',
+           'bit32.extract',
+           'bit32.lrotate',
+           'bit32.lshift',
+           'bit32.replace',
+           'bit32.rrotate',
+           'bit32.rshift'),
  'coroutine': ('coroutine.create',
+               'coroutine.isyieldable',
                'coroutine.resume',
                'coroutine.running',
                'coroutine.status',
                'coroutine.wrap',
                'coroutine.yield'),
  'debug': ('debug.debug',
-           'debug.getfenv',
            'debug.gethook',
            'debug.getinfo',
            'debug.getlocal',
            'debug.getmetatable',
            'debug.getregistry',
            'debug.getupvalue',
-           'debug.setfenv',
+           'debug.getuservalue',
            'debug.sethook',
            'debug.setlocal',
            'debug.setmetatable',
            'debug.setupvalue',
-           'debug.traceback'),
+           'debug.setuservalue',
+           'debug.traceback',
+           'debug.upvalueid',
+           'debug.upvaluejoin'),
  'io': ('io.close',
         'io.flush',
         'io.input',
@@ -71,17 +82,20 @@ MODULES = {'basic': ('_G',
         'io.output',
         'io.popen',
         'io.read',
+        'io.stderr',
+        'io.stdin',
+        'io.stdout',
         'io.tmpfile',
         'io.type',
         'io.write'),
  'math': ('math.abs',
           'math.acos',
           'math.asin',
-          'math.atan2',
           'math.atan',
+          'math.atan2',
           'math.ceil',
-          'math.cosh',
           'math.cos',
+          'math.cosh',
           'math.deg',
           'math.exp',
           'math.floor',
@@ -89,29 +103,34 @@ MODULES = {'basic': ('_G',
           'math.frexp',
           'math.huge',
           'math.ldexp',
-          'math.log10',
           'math.log',
           'math.max',
+          'math.maxinteger',
           'math.min',
+          'math.mininteger',
           'math.modf',
           'math.pi',
           'math.pow',
           'math.rad',
           'math.random',
           'math.randomseed',
-          'math.sinh',
           'math.sin',
+          'math.sinh',
           'math.sqrt',
+          'math.tan',
           'math.tanh',
-          'math.tan'),
- 'modules': ('module',
-             'require',
+          'math.tointeger',
+          'math.type',
+          'math.ult'),
+ 'modules': ('package.config',
              'package.cpath',
              'package.loaded',
              'package.loadlib',
              'package.path',
              'package.preload',
-             'package.seeall'),
+             'package.searchers',
+             'package.searchpath',
+             'require'),
  'os': ('os.clock',
         'os.date',
         'os.difftime',
@@ -133,18 +152,37 @@ MODULES = {'basic': ('_G',
             'string.len',
             'string.lower',
             'string.match',
+            'string.pack',
+            'string.packsize',
             'string.rep',
             'string.reverse',
             'string.sub',
+            'string.unpack',
             'string.upper'),
  'table': ('table.concat',
            'table.insert',
-           'table.maxn',
+           'table.move',
+           'table.pack',
            'table.remove',
-           'table.sort')}
+           'table.sort',
+           'table.unpack'),
+ 'utf8': ('utf8.char',
+          'utf8.charpattern',
+          'utf8.codepoint',
+          'utf8.codes',
+          'utf8.len',
+          'utf8.offset')}
 
-if __name__ == '__main__':
+if __name__ == '__main__':  # pragma: no cover
     import re
+    import sys
+
+    # urllib ends up wanting to import a module called 'math' -- if
+    # pygments/lexers is in the path, this ends badly.
+    for i in range(len(sys.path)-1, -1, -1):
+        if sys.path[i].endswith('/lexers'):
+            del sys.path[i]
+
     try:
         from urllib import urlopen
     except ImportError:
@@ -195,7 +233,7 @@ if __name__ == '__main__':
 
     def get_newest_version():
         f = urlopen('http://www.lua.org/manual/')
-        r = re.compile(r'^<A HREF="(\d\.\d)/">Lua \1</A>')
+        r = re.compile(r'^<A HREF="(\d\.\d)/">(Lua )?\1</A>')
         for line in f:
             m = r.match(line)
             if m is not None:
@@ -203,7 +241,7 @@ if __name__ == '__main__':
 
     def get_lua_functions(version):
         f = urlopen('http://www.lua.org/manual/%s/' % version)
-        r = re.compile(r'^<A HREF="manual.html#pdf-(.+)">\1</A>')
+        r = re.compile(r'^<A HREF="manual.html#pdf-(?!lua|LUA)([^:]+)">\1</A>')
         functions = []
         for line in f:
             m = r.match(line)
@@ -235,15 +273,22 @@ if __name__ == '__main__':
 
     def run():
         version = get_newest_version()
-        print('> Downloading function index for Lua %s' % version)
-        functions = get_lua_functions(version)
-        print('> %d functions found:' % len(functions))
+        functions = set()
+        for v in ('5.2', version):
+            print('> Downloading function index for Lua %s' % v)
+            f = get_lua_functions(v)
+            print('> %d functions found, %d new:' %
+                  (len(f), len(set(f) - functions)))
+            functions |= set(f)
+
+        functions = sorted(functions)
 
         modules = {}
         for full_function_name in functions:
             print('>> %s' % full_function_name)
             m = get_function_module(full_function_name)
             modules.setdefault(m, []).append(full_function_name)
+        modules = {k: tuple(v) for k, v in modules.iteritems()}
 
         regenerate(__file__, modules)
 

packages/wakatime/packages/pygments/lexers/_mapping.py

@@ -9,21 +9,27 @@
 
     Do not alter the LEXERS dictionary by hand.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2014, 2016 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 from __future__ import print_function
 
 LEXERS = {
-    'ABAPLexer': ('pygments.lexers.business', 'ABAP', ('abap',), ('*.abap',), ('text/x-abap',)),
+    'ABAPLexer': ('pygments.lexers.business', 'ABAP', ('abap',), ('*.abap', '*.ABAP'), ('text/x-abap',)),
     'APLLexer': ('pygments.lexers.apl', 'APL', ('apl',), ('*.apl',), ()),
+    'AbnfLexer': ('pygments.lexers.grammar_notation', 'ABNF', ('abnf',), ('*.abnf',), ('text/x-abnf',)),
     'ActionScript3Lexer': ('pygments.lexers.actionscript', 'ActionScript 3', ('as3', 'actionscript3'), ('*.as',), ('application/x-actionscript3', 'text/x-actionscript3', 'text/actionscript3')),
     'ActionScriptLexer': ('pygments.lexers.actionscript', 'ActionScript', ('as', 'actionscript'), ('*.as',), ('application/x-actionscript', 'text/x-actionscript', 'text/actionscript')),
     'AdaLexer': ('pygments.lexers.pascal', 'Ada', ('ada', 'ada95', 'ada2005'), ('*.adb', '*.ads', '*.ada'), ('text/x-ada',)),
+    'AdlLexer': ('pygments.lexers.archetype', 'ADL', ('adl',), ('*.adl', '*.adls', '*.adlf', '*.adlx'), ()),
     'AgdaLexer': ('pygments.lexers.haskell', 'Agda', ('agda',), ('*.agda',), ('text/x-agda',)),
+    'AheuiLexer': ('pygments.lexers.esoteric', 'Aheui', ('aheui',), ('*.aheui',), ()),
     'AlloyLexer': ('pygments.lexers.dsls', 'Alloy', ('alloy',), ('*.als',), ('text/x-alloy',)),
     'AmbientTalkLexer': ('pygments.lexers.ambient', 'AmbientTalk', ('at', 'ambienttalk', 'ambienttalk/2'), ('*.at',), ('text/x-ambienttalk',)),
+    'AmplLexer': ('pygments.lexers.ampl', 'Ampl', ('ampl',), ('*.run',), ()),
+    'Angular2HtmlLexer': ('pygments.lexers.templates', 'HTML + Angular2', ('html+ng2',), ('*.ng2',), ()),
+    'Angular2Lexer': ('pygments.lexers.templates', 'Angular2', ('ng2',), (), ()),
     'AntlrActionScriptLexer': ('pygments.lexers.parsers', 'ANTLR With ActionScript Target', ('antlr-as', 'antlr-actionscript'), ('*.G', '*.g'), ()),
     'AntlrCSharpLexer': ('pygments.lexers.parsers', 'ANTLR With C# Target', ('antlr-csharp', 'antlr-c#'), ('*.G', '*.g'), ()),
     'AntlrCppLexer': ('pygments.lexers.parsers', 'ANTLR With CPP Target', ('antlr-cpp',), ('*.G', '*.g'), ()),
@@ -35,29 +41,40 @@ LEXERS = {
     'AntlrRubyLexer': ('pygments.lexers.parsers', 'ANTLR With Ruby Target', ('antlr-ruby', 'antlr-rb'), ('*.G', '*.g'), ()),
     'ApacheConfLexer': ('pygments.lexers.configs', 'ApacheConf', ('apacheconf', 'aconf', 'apache'), ('.htaccess', 'apache.conf', 'apache2.conf'), ('text/x-apacheconf',)),
     'AppleScriptLexer': ('pygments.lexers.scripting', 'AppleScript', ('applescript',), ('*.applescript',), ()),
+    'ArduinoLexer': ('pygments.lexers.c_like', 'Arduino', ('arduino',), ('*.ino',), ('text/x-arduino',)),
     'AspectJLexer': ('pygments.lexers.jvm', 'AspectJ', ('aspectj',), ('*.aj',), ('text/x-aspectj',)),
     'AsymptoteLexer': ('pygments.lexers.graphics', 'Asymptote', ('asy', 'asymptote'), ('*.asy',), ('text/x-asymptote',)),
     'AutoItLexer': ('pygments.lexers.automation', 'AutoIt', ('autoit',), ('*.au3',), ('text/x-autoit',)),
     'AutohotkeyLexer': ('pygments.lexers.automation', 'autohotkey', ('ahk', 'autohotkey'), ('*.ahk', '*.ahkl'), ('text/x-autohotkey',)),
     'AwkLexer': ('pygments.lexers.textedit', 'Awk', ('awk', 'gawk', 'mawk', 'nawk'), ('*.awk',), ('application/x-awk',)),
     'BBCodeLexer': ('pygments.lexers.markup', 'BBCode', ('bbcode',), (), ('text/x-bbcode',)),
+    'BCLexer': ('pygments.lexers.algebra', 'BC', ('bc',), ('*.bc',), ()),
+    'BSTLexer': ('pygments.lexers.bibtex', 'BST', ('bst', 'bst-pybtex'), ('*.bst',), ()),
     'BaseMakefileLexer': ('pygments.lexers.make', 'Base Makefile', ('basemake',), (), ()),
-    'BashLexer': ('pygments.lexers.shell', 'Bash', ('bash', 'sh', 'ksh', 'shell'), ('*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass', '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'PKGBUILD'), ('application/x-sh', 'application/x-shellscript')),
-    'BashSessionLexer': ('pygments.lexers.shell', 'Bash Session', ('console',), ('*.sh-session',), ('application/x-shell-session',)),
+    'BashLexer': ('pygments.lexers.shell', 'Bash', ('bash', 'sh', 'ksh', 'zsh', 'shell'), ('*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass', '*.exheres-0', '*.exlib', '*.zsh', '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'zshrc', '.zshrc', 'PKGBUILD'), ('application/x-sh', 'application/x-shellscript')),
+    'BashSessionLexer': ('pygments.lexers.shell', 'Bash Session', ('console', 'shell-session'), ('*.sh-session', '*.shell-session'), ('application/x-shell-session', 'application/x-sh-session')),
     'BatchLexer': ('pygments.lexers.shell', 'Batchfile', ('bat', 'batch', 'dosbatch', 'winbatch'), ('*.bat', '*.cmd'), ('application/x-dos-batch',)),
     'BefungeLexer': ('pygments.lexers.esoteric', 'Befunge', ('befunge',), ('*.befunge',), ('application/x-befunge',)),
+    'BibTeXLexer': ('pygments.lexers.bibtex', 'BibTeX', ('bib', 'bibtex'), ('*.bib',), ('text/x-bibtex',)),
     'BlitzBasicLexer': ('pygments.lexers.basic', 'BlitzBasic', ('blitzbasic', 'b3d', 'bplus'), ('*.bb', '*.decls'), ('text/x-bb',)),
     'BlitzMaxLexer': ('pygments.lexers.basic', 'BlitzMax', ('blitzmax', 'bmax'), ('*.bmx',), ('text/x-bmx',)),
+    'BnfLexer': ('pygments.lexers.grammar_notation', 'BNF', ('bnf',), ('*.bnf',), ('text/x-bnf',)),
     'BooLexer': ('pygments.lexers.dotnet', 'Boo', ('boo',), ('*.boo',), ('text/x-boo',)),
+    'BoogieLexer': ('pygments.lexers.verification', 'Boogie', ('boogie',), ('*.bpl',), ()),
     'BrainfuckLexer': ('pygments.lexers.esoteric', 'Brainfuck', ('brainfuck', 'bf'), ('*.bf', '*.b'), ('application/x-brainfuck',)),
     'BroLexer': ('pygments.lexers.dsls', 'Bro', ('bro',), ('*.bro',), ()),
     'BugsLexer': ('pygments.lexers.modeling', 'BUGS', ('bugs', 'winbugs', 'openbugs'), ('*.bug',), ()),
+    'CAmkESLexer': ('pygments.lexers.esoteric', 'CAmkES', ('camkes', 'idl4'), ('*.camkes', '*.idl4'), ()),
     'CLexer': ('pygments.lexers.c_cpp', 'C', ('c',), ('*.c', '*.h', '*.idc'), ('text/x-chdr', 'text/x-csrc')),
     'CMakeLexer': ('pygments.lexers.make', 'CMake', ('cmake',), ('*.cmake', 'CMakeLists.txt'), ('text/x-cmake',)),
     'CObjdumpLexer': ('pygments.lexers.asm', 'c-objdump', ('c-objdump',), ('*.c-objdump',), ('text/x-c-objdump',)),
+    'CPSALexer': ('pygments.lexers.lisp', 'CPSA', ('cpsa',), ('*.cpsa',), ()),
     'CSharpAspxLexer': ('pygments.lexers.dotnet', 'aspx-cs', ('aspx-cs',), ('*.aspx', '*.asax', '*.ascx', '*.ashx', '*.asmx', '*.axd'), ()),
     'CSharpLexer': ('pygments.lexers.dotnet', 'C#', ('csharp', 'c#'), ('*.cs',), ('text/x-csharp',)),
     'Ca65Lexer': ('pygments.lexers.asm', 'ca65 assembler', ('ca65',), ('*.s',), ()),
+    'CadlLexer': ('pygments.lexers.archetype', 'cADL', ('cadl',), ('*.cadl',), ()),
+    'CapDLLexer': ('pygments.lexers.esoteric', 'CapDL', ('capdl',), ('*.cdl',), ()),
+    'CapnProtoLexer': ('pygments.lexers.capnproto', "Cap'n Proto", ('capnp',), ('*.capnp',), ()),
     'CbmBasicV2Lexer': ('pygments.lexers.basic', 'CBM BASIC V2', ('cbmbas',), ('*.bas',), ()),
     'CeylonLexer': ('pygments.lexers.jvm', 'Ceylon', ('ceylon',), ('*.ceylon',), ('text/x-ceylon',)),
     'Cfengine3Lexer': ('pygments.lexers.configs', 'CFEngine3', ('cfengine3', 'cf3'), ('*.cf',), ()),
@@ -67,8 +84,9 @@ LEXERS = {
     'CheetahJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Cheetah', ('js+cheetah', 'javascript+cheetah', 'js+spitfire', 'javascript+spitfire'), (), ('application/x-javascript+cheetah', 'text/x-javascript+cheetah', 'text/javascript+cheetah', 'application/x-javascript+spitfire', 'text/x-javascript+spitfire', 'text/javascript+spitfire')),
     'CheetahLexer': ('pygments.lexers.templates', 'Cheetah', ('cheetah', 'spitfire'), ('*.tmpl', '*.spt'), ('application/x-cheetah', 'application/x-spitfire')),
     'CheetahXmlLexer': ('pygments.lexers.templates', 'XML+Cheetah', ('xml+cheetah', 'xml+spitfire'), (), ('application/xml+cheetah', 'application/xml+spitfire')),
-    'CirruLexer': ('pygments.lexers.webmisc', 'Cirru', ('cirru',), ('*.cirru', '*.cr'), ('text/x-cirru',)),
+    'CirruLexer': ('pygments.lexers.webmisc', 'Cirru', ('cirru',), ('*.cirru',), ('text/x-cirru',)),
     'ClayLexer': ('pygments.lexers.c_like', 'Clay', ('clay',), ('*.clay',), ('text/x-clay',)),
+    'CleanLexer': ('pygments.lexers.clean', 'Clean', ('clean',), ('*.icl', '*.dcl'), ()),
     'ClojureLexer': ('pygments.lexers.jvm', 'Clojure', ('clojure', 'clj'), ('*.clj',), ('text/x-clojure', 'application/x-clojure')),
     'ClojureScriptLexer': ('pygments.lexers.jvm', 'ClojureScript', ('clojurescript', 'cljs'), ('*.cljs',), ('text/x-clojurescript', 'application/x-clojurescript')),
     'CobolFreeformatLexer': ('pygments.lexers.business', 'COBOLFree', ('cobolfree',), ('*.cbl', '*.CBL'), ()),
@@ -77,12 +95,18 @@ LEXERS = {
     'ColdfusionCFCLexer': ('pygments.lexers.templates', 'Coldfusion CFC', ('cfc',), ('*.cfc',), ()),
     'ColdfusionHtmlLexer': ('pygments.lexers.templates', 'Coldfusion HTML', ('cfm',), ('*.cfm', '*.cfml'), ('application/x-coldfusion',)),
     'ColdfusionLexer': ('pygments.lexers.templates', 'cfstatement', ('cfs',), (), ()),
-    'CommonLispLexer': ('pygments.lexers.lisp', 'Common Lisp', ('common-lisp', 'cl', 'lisp', 'elisp', 'emacs', 'emacs-lisp'), ('*.cl', '*.lisp', '*.el'), ('text/x-common-lisp',)),
+    'CommonLispLexer': ('pygments.lexers.lisp', 'Common Lisp', ('common-lisp', 'cl', 'lisp'), ('*.cl', '*.lisp'), ('text/x-common-lisp',)),
+    'ComponentPascalLexer': ('pygments.lexers.oberon', 'Component Pascal', ('componentpascal', 'cp'), ('*.cp', '*.cps'), ('text/x-component-pascal',)),
     'CoqLexer': ('pygments.lexers.theorem', 'Coq', ('coq',), ('*.v',), ('text/x-coq',)),
     'CppLexer': ('pygments.lexers.c_cpp', 'C++', ('cpp', 'c++'), ('*.cpp', '*.hpp', '*.c++', '*.h++', '*.cc', '*.hh', '*.cxx', '*.hxx', '*.C', '*.H', '*.cp', '*.CPP'), ('text/x-c++hdr', 'text/x-c++src')),
     'CppObjdumpLexer': ('pygments.lexers.asm', 'cpp-objdump', ('cpp-objdump', 'c++-objdumb', 'cxx-objdump'), ('*.cpp-objdump', '*.c++-objdump', '*.cxx-objdump'), ('text/x-cpp-objdump',)),
+    'CrmshLexer': ('pygments.lexers.dsls', 'Crmsh', ('crmsh', 'pcmk'), ('*.crmsh', '*.pcmk'), ()),
     'CrocLexer': ('pygments.lexers.d', 'Croc', ('croc',), ('*.croc',), ('text/x-crocsrc',)),
     'CryptolLexer': ('pygments.lexers.haskell', 'Cryptol', ('cryptol', 'cry'), ('*.cry',), ('text/x-cryptol',)),
+    'CrystalLexer': ('pygments.lexers.crystal', 'Crystal', ('cr', 'crystal'), ('*.cr',), ('text/x-crystal',)),
+    'CsoundDocumentLexer': ('pygments.lexers.csound', 'Csound Document', ('csound-document', 'csound-csd'), ('*.csd',), ()),
+    'CsoundOrchestraLexer': ('pygments.lexers.csound', 'Csound Orchestra', ('csound', 'csound-orc'), ('*.orc',), ()),
+    'CsoundScoreLexer': ('pygments.lexers.csound', 'Csound Score', ('csound-score', 'csound-sco'), ('*.sco',), ()),
     'CssDjangoLexer': ('pygments.lexers.templates', 'CSS+Django/Jinja', ('css+django', 'css+jinja'), (), ('text/css+django', 'text/css+jinja')),
     'CssErbLexer': ('pygments.lexers.templates', 'CSS+Ruby', ('css+erb', 'css+ruby'), (), ('text/css+ruby',)),
     'CssGenshiLexer': ('pygments.lexers.templates', 'CSS+Genshi Text', ('css+genshitext', 'css+genshi'), (), ('text/css+genshi',)),
@@ -97,7 +121,7 @@ LEXERS = {
     'DarcsPatchLexer': ('pygments.lexers.diff', 'Darcs Patch', ('dpatch',), ('*.dpatch', '*.darcspatch'), ()),
     'DartLexer': ('pygments.lexers.javascript', 'Dart', ('dart',), ('*.dart',), ('text/x-dart',)),
     'DebianControlLexer': ('pygments.lexers.installers', 'Debian Control file', ('control', 'debcontrol'), ('control',), ()),
-    'DelphiLexer': ('pygments.lexers.pascal', 'Delphi', ('delphi', 'pas', 'pascal', 'objectpascal'), ('*.pas',), ('text/x-pascal',)),
+    'DelphiLexer': ('pygments.lexers.pascal', 'Delphi', ('delphi', 'pas', 'pascal', 'objectpascal'), ('*.pas', '*.dpr'), ('text/x-pascal',)),
     'DgLexer': ('pygments.lexers.python', 'dg', ('dg',), ('*.dg',), ('text/x-dg',)),
     'DiffLexer': ('pygments.lexers.diff', 'Diff', ('diff', 'udiff'), ('*.diff', '*.patch'), ('text/x-diff', 'text/x-patch')),
     'DjangoLexer': ('pygments.lexers.templates', 'Django/Jinja', ('django', 'jinja'), (), ('application/x-django-templating', 'application/x-jinja')),
@@ -109,22 +133,31 @@ LEXERS = {
     'DylanLidLexer': ('pygments.lexers.dylan', 'DylanLID', ('dylan-lid', 'lid'), ('*.lid', '*.hdp'), ('text/x-dylan-lid',)),
     'ECLLexer': ('pygments.lexers.ecl', 'ECL', ('ecl',), ('*.ecl',), ('application/x-ecl',)),
     'ECLexer': ('pygments.lexers.c_like', 'eC', ('ec',), ('*.ec', '*.eh'), ('text/x-echdr', 'text/x-ecsrc')),
+    'EarlGreyLexer': ('pygments.lexers.javascript', 'Earl Grey', ('earl-grey', 'earlgrey', 'eg'), ('*.eg',), ('text/x-earl-grey',)),
+    'EasytrieveLexer': ('pygments.lexers.scripting', 'Easytrieve', ('easytrieve',), ('*.ezt', '*.mac'), ('text/x-easytrieve',)),
     'EbnfLexer': ('pygments.lexers.parsers', 'EBNF', ('ebnf',), ('*.ebnf',), ('text/x-ebnf',)),
     'EiffelLexer': ('pygments.lexers.eiffel', 'Eiffel', ('eiffel',), ('*.e',), ('text/x-eiffel',)),
     'ElixirConsoleLexer': ('pygments.lexers.erlang', 'Elixir iex session', ('iex',), (), ('text/x-elixir-shellsession',)),
     'ElixirLexer': ('pygments.lexers.erlang', 'Elixir', ('elixir', 'ex', 'exs'), ('*.ex', '*.exs'), ('text/x-elixir',)),
+    'ElmLexer': ('pygments.lexers.elm', 'Elm', ('elm',), ('*.elm',), ('text/x-elm',)),
+    'EmacsLispLexer': ('pygments.lexers.lisp', 'EmacsLisp', ('emacs', 'elisp', 'emacs-lisp'), ('*.el',), ('text/x-elisp', 'application/x-elisp')),
     'ErbLexer': ('pygments.lexers.templates', 'ERB', ('erb',), (), ('application/x-ruby-templating',)),
     'ErlangLexer': ('pygments.lexers.erlang', 'Erlang', ('erlang',), ('*.erl', '*.hrl', '*.es', '*.escript'), ('text/x-erlang',)),
     'ErlangShellLexer': ('pygments.lexers.erlang', 'Erlang erl session', ('erl',), ('*.erl-sh',), ('text/x-erl-shellsession',)),
     'EvoqueHtmlLexer': ('pygments.lexers.templates', 'HTML+Evoque', ('html+evoque',), ('*.html',), ('text/html+evoque',)),
     'EvoqueLexer': ('pygments.lexers.templates', 'Evoque', ('evoque',), ('*.evoque',), ('application/x-evoque',)),
     'EvoqueXmlLexer': ('pygments.lexers.templates', 'XML+Evoque', ('xml+evoque',), ('*.xml',), ('application/xml+evoque',)),
+    'EzhilLexer': ('pygments.lexers.ezhil', 'Ezhil', ('ezhil',), ('*.n',), ('text/x-ezhil',)),
     'FSharpLexer': ('pygments.lexers.dotnet', 'FSharp', ('fsharp',), ('*.fs', '*.fsi'), ('text/x-fsharp',)),
     'FactorLexer': ('pygments.lexers.factor', 'Factor', ('factor',), ('*.factor',), ('text/x-factor',)),
     'FancyLexer': ('pygments.lexers.ruby', 'Fancy', ('fancy', 'fy'), ('*.fy', '*.fancypack'), ('text/x-fancysrc',)),
     'FantomLexer': ('pygments.lexers.fantom', 'Fantom', ('fan',), ('*.fan',), ('application/x-fantom',)),
     'FelixLexer': ('pygments.lexers.felix', 'Felix', ('felix', 'flx'), ('*.flx', '*.flxh'), ('text/x-felix',)),
-    'FortranLexer': ('pygments.lexers.fortran', 'Fortran', ('fortran',), ('*.f', '*.f90', '*.F', '*.F90'), ('text/x-fortran',)),
+    'FishShellLexer': ('pygments.lexers.shell', 'Fish', ('fish', 'fishshell'), ('*.fish', '*.load'), ('application/x-fish',)),
+    'FlatlineLexer': ('pygments.lexers.dsls', 'Flatline', ('flatline',), (), ('text/x-flatline',)),
+    'ForthLexer': ('pygments.lexers.forth', 'Forth', ('forth',), ('*.frt', '*.fs'), ('application/x-forth',)),
+    'FortranFixedLexer': ('pygments.lexers.fortran', 'FortranFixed', ('fortranfixed',), ('*.f', '*.F'), ()),
+    'FortranLexer': ('pygments.lexers.fortran', 'Fortran', ('fortran',), ('*.f03', '*.f90', '*.F03', '*.F90'), ('text/x-fortran',)),
     'FoxProLexer': ('pygments.lexers.foxpro', 'FoxPro', ('foxpro', 'vfp', 'clipper', 'xbase'), ('*.PRG', '*.prg'), ()),
     'GAPLexer': ('pygments.lexers.algebra', 'GAP', ('gap',), ('*.g', '*.gd', '*.gi', '*.gap'), ()),
     'GLShaderLexer': ('pygments.lexers.graphics', 'GLSL', ('glsl',), ('*.vert', '*.frag', '*.geo'), ('text/x-glslsrc',)),
@@ -140,12 +173,14 @@ LEXERS = {
     'GosuLexer': ('pygments.lexers.jvm', 'Gosu', ('gosu',), ('*.gs', '*.gsx', '*.gsp', '*.vark'), ('text/x-gosu',)),
     'GosuTemplateLexer': ('pygments.lexers.jvm', 'Gosu Template', ('gst',), ('*.gst',), ('text/x-gosu-template',)),
     'GroffLexer': ('pygments.lexers.markup', 'Groff', ('groff', 'nroff', 'man'), ('*.[1234567]', '*.man'), ('application/x-troff', 'text/troff')),
-    'GroovyLexer': ('pygments.lexers.jvm', 'Groovy', ('groovy',), ('*.groovy',), ('text/x-groovy',)),
+    'GroovyLexer': ('pygments.lexers.jvm', 'Groovy', ('groovy',), ('*.groovy', '*.gradle'), ('text/x-groovy',)),
     'HamlLexer': ('pygments.lexers.html', 'Haml', ('haml',), ('*.haml',), ('text/x-haml',)),
     'HandlebarsHtmlLexer': ('pygments.lexers.templates', 'HTML+Handlebars', ('html+handlebars',), ('*.handlebars', '*.hbs'), ('text/html+handlebars', 'text/x-handlebars-template')),
     'HandlebarsLexer': ('pygments.lexers.templates', 'Handlebars', ('handlebars',), (), ()),
     'HaskellLexer': ('pygments.lexers.haskell', 'Haskell', ('haskell', 'hs'), ('*.hs',), ('text/x-haskell',)),
     'HaxeLexer': ('pygments.lexers.haxe', 'Haxe', ('hx', 'haxe', 'hxsl'), ('*.hx', '*.hxsl'), ('text/haxe', 'text/x-haxe', 'text/x-hx')),
+    'HexdumpLexer': ('pygments.lexers.hexdump', 'Hexdump', ('hexdump',), (), ()),
+    'HsailLexer': ('pygments.lexers.asm', 'HSAIL', ('hsail', 'hsa'), ('*.hsail',), ('text/x-hsail',)),
     'HtmlDjangoLexer': ('pygments.lexers.templates', 'HTML+Django/Jinja', ('html+django', 'html+jinja', 'htmldjango'), (), ('text/html+django', 'text/html+jinja')),
     'HtmlGenshiLexer': ('pygments.lexers.templates', 'HTML+Genshi', ('html+genshi', 'html+kid'), (), ('text/html+genshi',)),
     'HtmlLexer': ('pygments.lexers.html', 'HTML', ('html',), ('*.html', '*.htm', '*.xhtml', '*.xslt'), ('text/html', 'application/xhtml+xml')),
@@ -161,26 +196,30 @@ LEXERS = {
     'Inform6Lexer': ('pygments.lexers.int_fiction', 'Inform 6', ('inform6', 'i6'), ('*.inf',), ()),
     'Inform6TemplateLexer': ('pygments.lexers.int_fiction', 'Inform 6 template', ('i6t',), ('*.i6t',), ()),
     'Inform7Lexer': ('pygments.lexers.int_fiction', 'Inform 7', ('inform7', 'i7'), ('*.ni', '*.i7x'), ()),
-    'IniLexer': ('pygments.lexers.configs', 'INI', ('ini', 'cfg', 'dosini'), ('*.ini', '*.cfg'), ('text/x-ini',)),
+    'IniLexer': ('pygments.lexers.configs', 'INI', ('ini', 'cfg', 'dosini'), ('*.ini', '*.cfg', '*.inf'), ('text/x-ini', 'text/inf')),
     'IoLexer': ('pygments.lexers.iolang', 'Io', ('io',), ('*.io',), ('text/x-iosrc',)),
     'IokeLexer': ('pygments.lexers.jvm', 'Ioke', ('ioke', 'ik'), ('*.ik',), ('text/x-iokesrc',)),
     'IrcLogsLexer': ('pygments.lexers.textfmts', 'IRC logs', ('irc',), ('*.weechatlog',), ('text/x-irclog',)),
     'IsabelleLexer': ('pygments.lexers.theorem', 'Isabelle', ('isabelle',), ('*.thy',), ('text/x-isabelle',)),
-    'JadeLexer': ('pygments.lexers.html', 'Jade', ('jade',), ('*.jade',), ('text/x-jade',)),
+    'JLexer': ('pygments.lexers.j', 'J', ('j',), ('*.ijs',), ('text/x-j',)),
     'JagsLexer': ('pygments.lexers.modeling', 'JAGS', ('jags',), ('*.jag', '*.bug'), ()),
     'JasminLexer': ('pygments.lexers.jvm', 'Jasmin', ('jasmin', 'jasminxt'), ('*.j',), ()),
     'JavaLexer': ('pygments.lexers.jvm', 'Java', ('java',), ('*.java',), ('text/x-java',)),
     'JavascriptDjangoLexer': ('pygments.lexers.templates', 'JavaScript+Django/Jinja', ('js+django', 'javascript+django', 'js+jinja', 'javascript+jinja'), (), ('application/x-javascript+django', 'application/x-javascript+jinja', 'text/x-javascript+django', 'text/x-javascript+jinja', 'text/javascript+django', 'text/javascript+jinja')),
     'JavascriptErbLexer': ('pygments.lexers.templates', 'JavaScript+Ruby', ('js+erb', 'javascript+erb', 'js+ruby', 'javascript+ruby'), (), ('application/x-javascript+ruby', 'text/x-javascript+ruby', 'text/javascript+ruby')),
     'JavascriptGenshiLexer': ('pygments.lexers.templates', 'JavaScript+Genshi Text', ('js+genshitext', 'js+genshi', 'javascript+genshitext', 'javascript+genshi'), (), ('application/x-javascript+genshi', 'text/x-javascript+genshi', 'text/javascript+genshi')),
-    'JavascriptLexer': ('pygments.lexers.javascript', 'JavaScript', ('js', 'javascript'), ('*.js',), ('application/javascript', 'application/x-javascript', 'text/x-javascript', 'text/javascript')),
+    'JavascriptLexer': ('pygments.lexers.javascript', 'JavaScript', ('js', 'javascript'), ('*.js', '*.jsm'), ('application/javascript', 'application/x-javascript', 'text/x-javascript', 'text/javascript')),
     'JavascriptPhpLexer': ('pygments.lexers.templates', 'JavaScript+PHP', ('js+php', 'javascript+php'), (), ('application/x-javascript+php', 'text/x-javascript+php', 'text/javascript+php')),
     'JavascriptSmartyLexer': ('pygments.lexers.templates', 'JavaScript+Smarty', ('js+smarty', 'javascript+smarty'), (), ('application/x-javascript+smarty', 'text/x-javascript+smarty', 'text/javascript+smarty')),
+    'JclLexer': ('pygments.lexers.scripting', 'JCL', ('jcl',), ('*.jcl',), ('text/x-jcl',)),
+    'JsgfLexer': ('pygments.lexers.grammar_notation', 'JSGF', ('jsgf',), ('*.jsgf',), ('application/jsgf', 'application/x-jsgf', 'text/jsgf')),
+    'JsonBareObjectLexer': ('pygments.lexers.data', 'JSONBareObject', ('json-object',), (), ('application/json-object',)),
     'JsonLdLexer': ('pygments.lexers.data', 'JSON-LD', ('jsonld', 'json-ld'), ('*.jsonld',), ('application/ld+json',)),
     'JsonLexer': ('pygments.lexers.data', 'JSON', ('json',), ('*.json',), ('application/json',)),
     'JspLexer': ('pygments.lexers.templates', 'Java Server Page', ('jsp',), ('*.jsp',), ('application/x-jsp',)),
     'JuliaConsoleLexer': ('pygments.lexers.julia', 'Julia console', ('jlcon',), (), ()),
     'JuliaLexer': ('pygments.lexers.julia', 'Julia', ('julia', 'jl'), ('*.jl',), ('text/x-julia', 'application/x-julia')),
+    'JuttleLexer': ('pygments.lexers.javascript', 'Juttle', ('juttle', 'juttle'), ('*.juttle',), ('application/juttle', 'application/x-juttle', 'text/x-juttle', 'text/juttle')),
     'KalLexer': ('pygments.lexers.javascript', 'Kal', ('kal',), ('*.kal',), ('text/kal', 'application/kal')),
     'KconfigLexer': ('pygments.lexers.configs', 'Kconfig', ('kconfig', 'menuconfig', 'linux-config', 'kernel-config'), ('Kconfig', '*Config.in*', 'external.in*', 'standard-modules.in'), ('text/x-kconfig',)),
     'KokaLexer': ('pygments.lexers.haskell', 'Koka', ('koka',), ('*.kk', '*.kki'), ('text/x-koka',)),
@@ -192,6 +231,7 @@ LEXERS = {
     'LassoLexer': ('pygments.lexers.javascript', 'Lasso', ('lasso', 'lassoscript'), ('*.lasso', '*.lasso[89]'), ('text/x-lasso',)),
     'LassoXmlLexer': ('pygments.lexers.templates', 'XML+Lasso', ('xml+lasso',), (), ('application/xml+lasso',)),
     'LeanLexer': ('pygments.lexers.theorem', 'Lean', ('lean',), ('*.lean',), ('text/x-lean',)),
+    'LessCssLexer': ('pygments.lexers.css', 'LessCss', ('less',), ('*.less',), ('text/x-less-css',)),
     'LighttpdConfLexer': ('pygments.lexers.configs', 'Lighttpd configuration file', ('lighty', 'lighttpd'), (), ('text/x-lighttpd-conf',)),
     'LimboLexer': ('pygments.lexers.inferno', 'Limbo', ('limbo',), ('*.b',), ('text/limbo',)),
     'LiquidLexer': ('pygments.lexers.templates', 'liquid', ('liquid',), ('*.liquid',), ()),
@@ -205,6 +245,7 @@ LEXERS = {
     'LogtalkLexer': ('pygments.lexers.prolog', 'Logtalk', ('logtalk',), ('*.lgt', '*.logtalk'), ('text/x-logtalk',)),
     'LuaLexer': ('pygments.lexers.scripting', 'Lua', ('lua',), ('*.lua', '*.wlua'), ('text/x-lua', 'application/x-lua')),
     'MOOCodeLexer': ('pygments.lexers.scripting', 'MOOCode', ('moocode', 'moo'), ('*.moo',), ('text/x-moocode',)),
+    'MSDOSSessionLexer': ('pygments.lexers.shell', 'MSDOS Session', ('doscon',), (), ()),
     'MakefileLexer': ('pygments.lexers.make', 'Makefile', ('make', 'makefile', 'mf', 'bsdmake'), ('*.mak', '*.mk', 'Makefile', 'makefile', 'Makefile.*', 'GNUmakefile'), ('text/x-makefile',)),
     'MakoCssLexer': ('pygments.lexers.templates', 'CSS+Mako', ('css+mako',), (), ('text/css+mako',)),
     'MakoHtmlLexer': ('pygments.lexers.templates', 'HTML+Mako', ('html+mako',), (), ('text/html+mako',)),
@@ -212,6 +253,7 @@ LEXERS = {
     'MakoLexer': ('pygments.lexers.templates', 'Mako', ('mako',), ('*.mao',), ('application/x-mako',)),
     'MakoXmlLexer': ('pygments.lexers.templates', 'XML+Mako', ('xml+mako',), (), ('application/xml+mako',)),
     'MaqlLexer': ('pygments.lexers.business', 'MAQL', ('maql',), ('*.maql',), ('text/x-gooddata-maql', 'application/x-gooddata-maql')),
+    'MarkdownLexer': ('pygments.lexers.markup', 'markdown', ('md',), ('*.md',), ('text/x-markdown',)),
     'MaskLexer': ('pygments.lexers.javascript', 'Mask', ('mask',), ('*.mask',), ('text/x-mask',)),
     'MasonLexer': ('pygments.lexers.templates', 'Mason', ('mason',), ('*.m', '*.mhtml', '*.mc', '*.mi', 'autohandler', 'dhandler'), ('application/x-mason',)),
     'MathematicaLexer': ('pygments.lexers.algebra', 'Mathematica', ('mathematica', 'mma', 'nb'), ('*.nb', '*.cdf', '*.nbp', '*.ma'), ('application/mathematica', 'application/vnd.wolfram.mathematica', 'application/vnd.wolfram.mathematica.package', 'application/vnd.wolfram.cdf')),
@@ -219,9 +261,10 @@ LEXERS = {
     'MatlabSessionLexer': ('pygments.lexers.matlab', 'Matlab session', ('matlabsession',), (), ()),
     'MiniDLexer': ('pygments.lexers.d', 'MiniD', ('minid',), (), ('text/x-minidsrc',)),
     'ModelicaLexer': ('pygments.lexers.modeling', 'Modelica', ('modelica',), ('*.mo',), ('text/x-modelica',)),
-    'Modula2Lexer': ('pygments.lexers.pascal', 'Modula-2', ('modula2', 'm2'), ('*.def', '*.mod'), ('text/x-modula2',)),
+    'Modula2Lexer': ('pygments.lexers.modula2', 'Modula-2', ('modula2', 'm2'), ('*.def', '*.mod'), ('text/x-modula2',)),
     'MoinWikiLexer': ('pygments.lexers.markup', 'MoinMoin/Trac Wiki markup', ('trac-wiki', 'moin'), (), ('text/x-trac-wiki',)),
     'MonkeyLexer': ('pygments.lexers.basic', 'Monkey', ('monkey',), ('*.monkey',), ('text/x-monkey',)),
+    'MonteLexer': ('pygments.lexers.monte', 'Monte', ('monte',), ('*.mt',), ()),
     'MoonScriptLexer': ('pygments.lexers.scripting', 'MoonScript', ('moon', 'moonscript'), ('*.moon',), ('text/x-moonscript', 'application/x-moonscript')),
     'MozPreprocCssLexer': ('pygments.lexers.markup', 'CSS+mozpreproc', ('css+mozpreproc',), ('*.css.in',), ()),
     'MozPreprocHashLexer': ('pygments.lexers.markup', 'mozhashpreproc', ('mozhashpreproc',), (), ()),
@@ -238,17 +281,19 @@ LEXERS = {
     'MyghtyJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Myghty', ('js+myghty', 'javascript+myghty'), (), ('application/x-javascript+myghty', 'text/x-javascript+myghty', 'text/javascript+mygthy')),
     'MyghtyLexer': ('pygments.lexers.templates', 'Myghty', ('myghty',), ('*.myt', 'autodelegate'), ('application/x-myghty',)),
     'MyghtyXmlLexer': ('pygments.lexers.templates', 'XML+Myghty', ('xml+myghty',), (), ('application/xml+myghty',)),
+    'NCLLexer': ('pygments.lexers.ncl', 'NCL', ('ncl',), ('*.ncl',), ('text/ncl',)),
     'NSISLexer': ('pygments.lexers.installers', 'NSIS', ('nsis', 'nsi', 'nsh'), ('*.nsi', '*.nsh'), ('text/x-nsis',)),
     'NasmLexer': ('pygments.lexers.asm', 'NASM', ('nasm',), ('*.asm', '*.ASM'), ('text/x-nasm',)),
     'NasmObjdumpLexer': ('pygments.lexers.asm', 'objdump-nasm', ('objdump-nasm',), ('*.objdump-intel',), ('text/x-nasm-objdump',)),
     'NemerleLexer': ('pygments.lexers.dotnet', 'Nemerle', ('nemerle',), ('*.n',), ('text/x-nemerle',)),
     'NesCLexer': ('pygments.lexers.c_like', 'nesC', ('nesc',), ('*.nc',), ('text/x-nescsrc',)),
-    'NewLispLexer': ('pygments.lexers.lisp', 'NewLisp', ('newlisp',), ('*.lsp', '*.nl'), ('text/x-newlisp', 'application/x-newlisp')),
+    'NewLispLexer': ('pygments.lexers.lisp', 'NewLisp', ('newlisp',), ('*.lsp', '*.nl', '*.kif'), ('text/x-newlisp', 'application/x-newlisp')),
     'NewspeakLexer': ('pygments.lexers.smalltalk', 'Newspeak', ('newspeak',), ('*.ns2',), ('text/x-newspeak',)),
-    'NginxConfLexer': ('pygments.lexers.configs', 'Nginx configuration file', ('nginx',), (), ('text/x-nginx-conf',)),
-    'NimrodLexer': ('pygments.lexers.nimrod', 'Nimrod', ('nimrod', 'nim'), ('*.nim', '*.nimrod'), ('text/x-nimrod',)),
+    'NginxConfLexer': ('pygments.lexers.configs', 'Nginx configuration file', ('nginx',), ('nginx.conf',), ('text/x-nginx-conf',)),
+    'NimrodLexer': ('pygments.lexers.nimrod', 'Nimrod', ('nim', 'nimrod'), ('*.nim', '*.nimrod'), ('text/x-nim',)),
     'NitLexer': ('pygments.lexers.nit', 'Nit', ('nit',), ('*.nit',), ()),
     'NixLexer': ('pygments.lexers.nix', 'Nix', ('nixos', 'nix'), ('*.nix',), ('text/x-nix',)),
+    'NuSMVLexer': ('pygments.lexers.smv', 'NuSMV', ('nusmv',), ('*.smv',), ()),
     'NumPyLexer': ('pygments.lexers.python', 'NumPy', ('numpy',), (), ()),
     'ObjdumpLexer': ('pygments.lexers.asm', 'objdump', ('objdump',), ('*.objdump',), ('text/x-objdump',)),
     'ObjectiveCLexer': ('pygments.lexers.objective', 'Objective-C', ('objective-c', 'objectivec', 'obj-c', 'objc'), ('*.m', '*.h'), ('text/x-objective-c',)),
@@ -256,25 +301,32 @@ LEXERS = {
     'ObjectiveJLexer': ('pygments.lexers.javascript', 'Objective-J', ('objective-j', 'objectivej', 'obj-j', 'objj'), ('*.j',), ('text/x-objective-j',)),
     'OcamlLexer': ('pygments.lexers.ml', 'OCaml', ('ocaml',), ('*.ml', '*.mli', '*.mll', '*.mly'), ('text/x-ocaml',)),
     'OctaveLexer': ('pygments.lexers.matlab', 'Octave', ('octave',), ('*.m',), ('text/octave',)),
+    'OdinLexer': ('pygments.lexers.archetype', 'ODIN', ('odin',), ('*.odin',), ('text/odin',)),
     'OocLexer': ('pygments.lexers.ooc', 'Ooc', ('ooc',), ('*.ooc',), ('text/x-ooc',)),
     'OpaLexer': ('pygments.lexers.ml', 'Opa', ('opa',), ('*.opa',), ('text/x-opa',)),
     'OpenEdgeLexer': ('pygments.lexers.business', 'OpenEdge ABL', ('openedge', 'abl', 'progress'), ('*.p', '*.cls'), ('text/x-openedge', 'application/x-openedge')),
+    'PacmanConfLexer': ('pygments.lexers.configs', 'PacmanConf', ('pacmanconf',), ('pacman.conf',), ()),
     'PanLexer': ('pygments.lexers.dsls', 'Pan', ('pan',), ('*.pan',), ()),
+    'ParaSailLexer': ('pygments.lexers.parasail', 'ParaSail', ('parasail',), ('*.psi', '*.psl'), ('text/x-parasail',)),
     'PawnLexer': ('pygments.lexers.pawn', 'Pawn', ('pawn',), ('*.p', '*.pwn', '*.inc'), ('text/x-pawn',)),
     'Perl6Lexer': ('pygments.lexers.perl', 'Perl6', ('perl6', 'pl6'), ('*.pl', '*.pm', '*.nqp', '*.p6', '*.6pl', '*.p6l', '*.pl6', '*.6pm', '*.p6m', '*.pm6', '*.t'), ('text/x-perl6', 'application/x-perl6')),
     'PerlLexer': ('pygments.lexers.perl', 'Perl', ('perl', 'pl'), ('*.pl', '*.pm', '*.t'), ('text/x-perl', 'application/x-perl')),
     'PhpLexer': ('pygments.lexers.php', 'PHP', ('php', 'php3', 'php4', 'php5'), ('*.php', '*.php[345]', '*.inc'), ('text/x-php',)),
     'PigLexer': ('pygments.lexers.jvm', 'Pig', ('pig',), ('*.pig',), ('text/x-pig',)),
     'PikeLexer': ('pygments.lexers.c_like', 'Pike', ('pike',), ('*.pike', '*.pmod'), ('text/x-pike',)),
+    'PkgConfigLexer': ('pygments.lexers.configs', 'PkgConfig', ('pkgconfig',), ('*.pc',), ()),
     'PlPgsqlLexer': ('pygments.lexers.sql', 'PL/pgSQL', ('plpgsql',), (), ('text/x-plpgsql',)),
     'PostScriptLexer': ('pygments.lexers.graphics', 'PostScript', ('postscript', 'postscr'), ('*.ps', '*.eps'), ('application/postscript',)),
     'PostgresConsoleLexer': ('pygments.lexers.sql', 'PostgreSQL console (psql)', ('psql', 'postgresql-console', 'postgres-console'), (), ('text/x-postgresql-psql',)),
     'PostgresLexer': ('pygments.lexers.sql', 'PostgreSQL SQL dialect', ('postgresql', 'postgres'), (), ('text/x-postgresql',)),
     'PovrayLexer': ('pygments.lexers.graphics', 'POVRay', ('pov',), ('*.pov', '*.inc'), ('text/x-povray',)),
     'PowerShellLexer': ('pygments.lexers.shell', 'PowerShell', ('powershell', 'posh', 'ps1', 'psm1'), ('*.ps1', '*.psm1'), ('text/x-powershell',)),
+    'PowerShellSessionLexer': ('pygments.lexers.shell', 'PowerShell Session', ('ps1con',), (), ()),
+    'PraatLexer': ('pygments.lexers.praat', 'Praat', ('praat',), ('*.praat', '*.proc', '*.psc'), ()),
     'PrologLexer': ('pygments.lexers.prolog', 'Prolog', ('prolog',), ('*.ecl', '*.prolog', '*.pro', '*.pl'), ('text/x-prolog',)),
     'PropertiesLexer': ('pygments.lexers.configs', 'Properties', ('properties', 'jproperties'), ('*.properties',), ('text/x-java-properties',)),
     'ProtoBufLexer': ('pygments.lexers.dsls', 'Protocol Buffer', ('protobuf', 'proto'), ('*.proto',), ()),
+    'PugLexer': ('pygments.lexers.html', 'Pug', ('pug', 'jade'), ('*.pug', '*.jade'), ('text/x-pug', 'text/x-jade')),
     'PuppetLexer': ('pygments.lexers.dsls', 'Puppet', ('puppet',), ('*.pp',), ()),
     'PyPyLogLexer': ('pygments.lexers.console', 'PyPy Log', ('pypylog', 'pypy'), ('*.pypylog',), ('application/x-pypylog',)),
     'Python3Lexer': ('pygments.lexers.python', 'Python 3', ('python3', 'py3'), (), ('text/x-python3', 'application/x-python3')),
@@ -283,8 +335,10 @@ LEXERS = {
     'PythonLexer': ('pygments.lexers.python', 'Python', ('python', 'py', 'sage'), ('*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript', '*.tac', '*.sage'), ('text/x-python', 'application/x-python')),
     'PythonTracebackLexer': ('pygments.lexers.python', 'Python Traceback', ('pytb',), ('*.pytb',), ('text/x-python-traceback',)),
     'QBasicLexer': ('pygments.lexers.basic', 'QBasic', ('qbasic', 'basic'), ('*.BAS', '*.bas'), ('text/basic',)),
-    'QmlLexer': ('pygments.lexers.webmisc', 'QML', ('qml',), ('*.qml',), ('application/x-qml',)),
+    'QVToLexer': ('pygments.lexers.qvt', 'QVTO', ('qvto', 'qvt'), ('*.qvto',), ()),
+    'QmlLexer': ('pygments.lexers.webmisc', 'QML', ('qml', 'qbs'), ('*.qml', '*.qbs'), ('application/x-qml', 'application/x-qt.qbs+qml')),
     'RConsoleLexer': ('pygments.lexers.r', 'RConsole', ('rconsole', 'rout'), ('*.Rout',), ()),
+    'RNCCompactLexer': ('pygments.lexers.rnc', 'Relax-NG Compact', ('rnc', 'rng-compact'), ('*.rnc',), ()),
     'RPMSpecLexer': ('pygments.lexers.installers', 'RPMSpec', ('spec',), ('*.spec',), ('text/x-rpm-spec',)),
     'RacketLexer': ('pygments.lexers.lisp', 'Racket', ('racket', 'rkt'), ('*.rkt', '*.rktd', '*.rktl'), ('text/x-racket', 'application/x-racket')),
     'RagelCLexer': ('pygments.lexers.parsers', 'Ragel in C Host', ('ragel-c',), ('*.rl',), ()),
@@ -304,13 +358,17 @@ LEXERS = {
     'ResourceLexer': ('pygments.lexers.resource', 'ResourceBundle', ('resource', 'resourcebundle'), ('*.txt',), ()),
     'RexxLexer': ('pygments.lexers.scripting', 'Rexx', ('rexx', 'arexx'), ('*.rexx', '*.rex', '*.rx', '*.arexx'), ('text/x-rexx',)),
     'RhtmlLexer': ('pygments.lexers.templates', 'RHTML', ('rhtml', 'html+erb', 'html+ruby'), ('*.rhtml',), ('text/html+ruby',)),
+    'RoboconfGraphLexer': ('pygments.lexers.roboconf', 'Roboconf Graph', ('roboconf-graph',), ('*.graph',), ()),
+    'RoboconfInstancesLexer': ('pygments.lexers.roboconf', 'Roboconf Instances', ('roboconf-instances',), ('*.instances',), ()),
     'RobotFrameworkLexer': ('pygments.lexers.robotframework', 'RobotFramework', ('robotframework',), ('*.txt', '*.robot'), ('text/x-robotframework',)),
     'RqlLexer': ('pygments.lexers.sql', 'RQL', ('rql',), ('*.rql',), ('text/x-rql',)),
     'RslLexer': ('pygments.lexers.dsls', 'RSL', ('rsl',), ('*.rsl',), ('text/rsl',)),
     'RstLexer': ('pygments.lexers.markup', 'reStructuredText', ('rst', 'rest', 'restructuredtext'), ('*.rst', '*.rest'), ('text/x-rst', 'text/prs.fallenstein.rst')),
+    'RtsLexer': ('pygments.lexers.trafficscript', 'TrafficScript', ('rts', 'trafficscript'), ('*.rts',), ()),
     'RubyConsoleLexer': ('pygments.lexers.ruby', 'Ruby irb session', ('rbcon', 'irb'), (), ('text/x-ruby-shellsession',)),
-    'RubyLexer': ('pygments.lexers.ruby', 'Ruby', ('rb', 'ruby', 'duby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby'), ('text/x-ruby', 'application/x-ruby')),
-    'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust',), ('*.rs',), ('text/x-rustsrc',)),
+    'RubyLexer': ('pygments.lexers.ruby', 'Ruby', ('rb', 'ruby', 'duby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby', 'Gemfile'), ('text/x-ruby', 'application/x-ruby')),
+    'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust',), ('*.rs', '*.rs.in'), ('text/rust',)),
+    'SASLexer': ('pygments.lexers.sas', 'SAS', ('sas',), ('*.SAS', '*.sas'), ('text/x-sas', 'text/sas', 'application/x-sas')),
     'SLexer': ('pygments.lexers.r', 'S', ('splus', 's', 'r'), ('*.S', '*.R', '.Rhistory', '.Rprofile', '.Renviron'), ('text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile')),
     'SMLLexer': ('pygments.lexers.ml', 'Standard ML', ('sml',), ('*.sml', '*.sig', '*.fun'), ('text/x-standardml', 'application/x-standardml')),
     'SassLexer': ('pygments.lexers.css', 'Sass', ('sass',), ('*.sass',), ('text/x-sass',)),
@@ -319,12 +377,14 @@ LEXERS = {
     'SchemeLexer': ('pygments.lexers.lisp', 'Scheme', ('scheme', 'scm'), ('*.scm', '*.ss'), ('text/x-scheme', 'application/x-scheme')),
     'ScilabLexer': ('pygments.lexers.matlab', 'Scilab', ('scilab',), ('*.sci', '*.sce', '*.tst'), ('text/scilab',)),
     'ScssLexer': ('pygments.lexers.css', 'SCSS', ('scss',), ('*.scss',), ('text/x-scss',)),
-    'ShellSessionLexer': ('pygments.lexers.shell', 'Shell Session', ('shell-session',), ('*.shell-session',), ('application/x-sh-session',)),
+    'ShenLexer': ('pygments.lexers.lisp', 'Shen', ('shen',), ('*.shen',), ('text/x-shen', 'application/x-shen')),
+    'SilverLexer': ('pygments.lexers.verification', 'Silver', ('silver',), ('*.sil', '*.vpr'), ()),
     'SlimLexer': ('pygments.lexers.webmisc', 'Slim', ('slim',), ('*.slim',), ('text/x-slim',)),
     'SmaliLexer': ('pygments.lexers.dalvik', 'Smali', ('smali',), ('*.smali',), ('text/smali',)),
     'SmalltalkLexer': ('pygments.lexers.smalltalk', 'Smalltalk', ('smalltalk', 'squeak', 'st'), ('*.st',), ('text/x-smalltalk',)),
     'SmartyLexer': ('pygments.lexers.templates', 'Smarty', ('smarty',), ('*.tpl',), ('application/x-smarty',)),
     'SnobolLexer': ('pygments.lexers.snobol', 'Snobol', ('snobol',), ('*.snobol',), ('text/x-snobol',)),
+    'SnowballLexer': ('pygments.lexers.dsls', 'Snowball', ('snowball',), ('*.sbl',), ()),
     'SourcePawnLexer': ('pygments.lexers.pawn', 'SourcePawn', ('sp',), ('*.sp',), ('text/x-sourcepawn',)),
     'SourcesListLexer': ('pygments.lexers.installers', 'Debian Sourcelist', ('sourceslist', 'sources.list', 'debsources'), ('sources.list',), ()),
     'SparqlLexer': ('pygments.lexers.rdf', 'SPARQL', ('sparql',), ('*.rq', '*.sparql'), ('application/sparql-query',)),
@@ -333,21 +393,37 @@ LEXERS = {
     'SquidConfLexer': ('pygments.lexers.configs', 'SquidConf', ('squidconf', 'squid.conf', 'squid'), ('squid.conf',), ('text/x-squidconf',)),
     'SspLexer': ('pygments.lexers.templates', 'Scalate Server Page', ('ssp',), ('*.ssp',), ('application/x-ssp',)),
     'StanLexer': ('pygments.lexers.modeling', 'Stan', ('stan',), ('*.stan',), ()),
+    'StataLexer': ('pygments.lexers.stata', 'Stata', ('stata', 'do'), ('*.do', '*.ado'), ('text/x-stata', 'text/stata', 'application/x-stata')),
+    'SuperColliderLexer': ('pygments.lexers.supercollider', 'SuperCollider', ('sc', 'supercollider'), ('*.sc', '*.scd'), ('application/supercollider', 'text/supercollider')),
     'SwiftLexer': ('pygments.lexers.objective', 'Swift', ('swift',), ('*.swift',), ('text/x-swift',)),
     'SwigLexer': ('pygments.lexers.c_like', 'SWIG', ('swig',), ('*.swg', '*.i'), ('text/swig',)),
     'SystemVerilogLexer': ('pygments.lexers.hdl', 'systemverilog', ('systemverilog', 'sv'), ('*.sv', '*.svh'), ('text/x-systemverilog',)),
+    'TAPLexer': ('pygments.lexers.testing', 'TAP', ('tap',), ('*.tap',), ()),
     'Tads3Lexer': ('pygments.lexers.int_fiction', 'TADS 3', ('tads3',), ('*.t',), ()),
+    'TasmLexer': ('pygments.lexers.asm', 'TASM', ('tasm',), ('*.asm', '*.ASM', '*.tasm'), ('text/x-tasm',)),
     'TclLexer': ('pygments.lexers.tcl', 'Tcl', ('tcl',), ('*.tcl', '*.rvt'), ('text/x-tcl', 'text/x-script.tcl', 'application/x-tcl')),
     'TcshLexer': ('pygments.lexers.shell', 'Tcsh', ('tcsh', 'csh'), ('*.tcsh', '*.csh'), ('application/x-csh',)),
+    'TcshSessionLexer': ('pygments.lexers.shell', 'Tcsh Session', ('tcshcon',), (), ()),
     'TeaTemplateLexer': ('pygments.lexers.templates', 'Tea', ('tea',), ('*.tea',), ('text/x-tea',)),
+    'TermcapLexer': ('pygments.lexers.configs', 'Termcap', ('termcap',), ('termcap', 'termcap.src'), ()),
+    'TerminfoLexer': ('pygments.lexers.configs', 'Terminfo', ('terminfo',), ('terminfo', 'terminfo.src'), ()),
+    'TerraformLexer': ('pygments.lexers.configs', 'Terraform', ('terraform', 'tf'), ('*.tf',), ('application/x-tf', 'application/x-terraform')),
     'TexLexer': ('pygments.lexers.markup', 'TeX', ('tex', 'latex'), ('*.tex', '*.aux', '*.toc'), ('text/x-tex', 'text/x-latex')),
     'TextLexer': ('pygments.lexers.special', 'Text only', ('text',), ('*.txt',), ('text/plain',)),
+    'ThriftLexer': ('pygments.lexers.dsls', 'Thrift', ('thrift',), ('*.thrift',), ('application/x-thrift',)),
     'TodotxtLexer': ('pygments.lexers.textfmts', 'Todotxt', ('todotxt',), ('todo.txt', '*.todotxt'), ('text/x-todo',)),
+    'TransactSqlLexer': ('pygments.lexers.sql', 'Transact-SQL', ('tsql', 't-sql'), ('*.sql',), ('text/x-tsql',)),
     'TreetopLexer': ('pygments.lexers.parsers', 'Treetop', ('treetop',), ('*.treetop', '*.tt'), ()),
+    'TurtleLexer': ('pygments.lexers.rdf', 'Turtle', ('turtle',), ('*.ttl',), ('text/turtle', 'application/x-turtle')),
     'TwigHtmlLexer': ('pygments.lexers.templates', 'HTML+Twig', ('html+twig',), ('*.twig',), ('text/html+twig',)),
     'TwigLexer': ('pygments.lexers.templates', 'Twig', ('twig',), (), ('application/x-twig',)),
-    'TypeScriptLexer': ('pygments.lexers.javascript', 'TypeScript', ('ts',), ('*.ts',), ('text/x-typescript',)),
+    'TypeScriptLexer': ('pygments.lexers.javascript', 'TypeScript', ('ts', 'typescript'), ('*.ts', '*.tsx'), ('text/x-typescript',)),
+    'TypoScriptCssDataLexer': ('pygments.lexers.typoscript', 'TypoScriptCssData', ('typoscriptcssdata',), (), ()),
+    'TypoScriptHtmlDataLexer': ('pygments.lexers.typoscript', 'TypoScriptHtmlData', ('typoscripthtmldata',), (), ()),
+    'TypoScriptLexer': ('pygments.lexers.typoscript', 'TypoScript', ('typoscript',), ('*.ts', '*.txt'), ('text/x-typoscript',)),
     'UrbiscriptLexer': ('pygments.lexers.urbi', 'UrbiScript', ('urbiscript',), ('*.u',), ('application/x-urbiscript',)),
+    'VCLLexer': ('pygments.lexers.varnish', 'VCL', ('vcl',), ('*.vcl',), ('text/x-vclsrc',)),
+    'VCLSnippetLexer': ('pygments.lexers.varnish', 'VCLSnippets', ('vclsnippets', 'vclsnippet'), (), ('text/x-vclsnippet',)),
     'VCTreeStatusLexer': ('pygments.lexers.console', 'VCTreeStatus', ('vctreestatus',), (), ()),
     'VGLLexer': ('pygments.lexers.dsls', 'VGL', ('vgl',), ('*.rpf',), ()),
     'ValaLexer': ('pygments.lexers.c_like', 'Vala', ('vala', 'vapi'), ('*.vala', '*.vapi'), ('text/x-vala',)),
@@ -359,20 +435,25 @@ LEXERS = {
     'VerilogLexer': ('pygments.lexers.hdl', 'verilog', ('verilog', 'v'), ('*.v',), ('text/x-verilog',)),
     'VhdlLexer': ('pygments.lexers.hdl', 'vhdl', ('vhdl',), ('*.vhdl', '*.vhd'), ('text/x-vhdl',)),
     'VimLexer': ('pygments.lexers.textedit', 'VimL', ('vim',), ('*.vim', '.vimrc', '.exrc', '.gvimrc', '_vimrc', '_exrc', '_gvimrc', 'vimrc', 'gvimrc'), ('text/x-vim',)),
+    'WDiffLexer': ('pygments.lexers.diff', 'WDiff', ('wdiff',), ('*.wdiff',), ()),
+    'WhileyLexer': ('pygments.lexers.whiley', 'Whiley', ('whiley',), ('*.whiley',), ('text/x-whiley',)),
+    'X10Lexer': ('pygments.lexers.x10', 'X10', ('x10', 'xten'), ('*.x10',), ('text/x-x10',)),
     'XQueryLexer': ('pygments.lexers.webmisc', 'XQuery', ('xquery', 'xqy', 'xq', 'xql', 'xqm'), ('*.xqy', '*.xquery', '*.xq', '*.xql', '*.xqm'), ('text/xquery', 'application/xquery')),
     'XmlDjangoLexer': ('pygments.lexers.templates', 'XML+Django/Jinja', ('xml+django', 'xml+jinja'), (), ('application/xml+django', 'application/xml+jinja')),
     'XmlErbLexer': ('pygments.lexers.templates', 'XML+Ruby', ('xml+erb', 'xml+ruby'), (), ('application/xml+ruby',)),
     'XmlLexer': ('pygments.lexers.html', 'XML', ('xml',), ('*.xml', '*.xsl', '*.rss', '*.xslt', '*.xsd', '*.wsdl', '*.wsf'), ('text/xml', 'application/xml', 'image/svg+xml', 'application/rss+xml', 'application/atom+xml')),
     'XmlPhpLexer': ('pygments.lexers.templates', 'XML+PHP', ('xml+php',), (), ('application/xml+php',)),
     'XmlSmartyLexer': ('pygments.lexers.templates', 'XML+Smarty', ('xml+smarty',), (), ('application/xml+smarty',)),
+    'XorgLexer': ('pygments.lexers.xorg', 'Xorg', ('xorg.conf',), ('xorg.conf',), ()),
     'XsltLexer': ('pygments.lexers.html', 'XSLT', ('xslt',), ('*.xsl', '*.xslt', '*.xpl'), ('application/xsl+xml', 'application/xslt+xml')),
     'XtendLexer': ('pygments.lexers.jvm', 'Xtend', ('xtend',), ('*.xtend',), ('text/x-xtend',)),
+    'XtlangLexer': ('pygments.lexers.lisp', 'xtlang', ('extempore',), ('*.xtm',), ()),
     'YamlJinjaLexer': ('pygments.lexers.templates', 'YAML+Jinja', ('yaml+jinja', 'salt', 'sls'), ('*.sls',), ('text/x-yaml+jinja', 'text/x-sls')),
     'YamlLexer': ('pygments.lexers.data', 'YAML', ('yaml',), ('*.yaml', '*.yml'), ('text/x-yaml',)),
     'ZephirLexer': ('pygments.lexers.php', 'Zephir', ('zephir',), ('*.zep',), ()),
 }
 
-if __name__ == '__main__':
+if __name__ == '__main__':  # pragma: no cover
     import sys
     import os
 
@@ -401,6 +482,13 @@ if __name__ == '__main__':
     # extract useful sourcecode from this file
     with open(__file__) as fp:
         content = fp.read()
+        # replace crnl to nl for Windows.
+        #
+        # Note that, originally, contributers should keep nl of master
+        # repository, for example by using some kind of automatic
+        # management EOL, like `EolExtension
+        #  <https://www.mercurial-scm.org/wiki/EolExtension>`.
+        content = content.replace("\r\n", "\n")
     header = content[:content.find('LEXERS = {')]
     footer = content[content.find("if __name__ == '__main__':"):]
 

packages/wakatime/packages/pygments/lexers/_mql_builtins.py

@@ -5,7 +5,7 @@
 
     Builtins for the MqlLexer.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 types = (

packages/wakatime/packages/pygments/lexers/_openedge_builtins.py

@@ -5,7 +5,7 @@
 
     Builtin list for the OpenEdgeLexer.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/_php_builtins.py

@@ -12,7 +12,7 @@
              internet connection. don't run that at home, use
              a server ;-)
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -4672,7 +4672,8 @@ MODULES = {'.NET': ('dotnet_load',),
            'xdiff_string_patch',
            'xdiff_string_rabdiff')}
 
-if __name__ == '__main__':
+
+if __name__ == '__main__':  # pragma: no cover
     import glob
     import os
     import pprint

packages/wakatime/packages/pygments/lexers/_postgres_builtins.py

@@ -5,7 +5,7 @@
 
     Self-updating data files for PostgreSQL lexer.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -507,7 +507,8 @@ PLPGSQL_KEYWORDS = (
     'RETURN', 'REVERSE', 'SQLSTATE', 'WHILE',
 )
 
-if __name__ == '__main__':
+
+if __name__ == '__main__':  # pragma: no cover
     import re
     try:
         from urllib import urlopen

packages/wakatime/packages/pygments/lexers/_scilab_builtins.py

@@ -5,7 +5,7 @@
 
     Builtin list for the ScilabLexer.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -3051,7 +3051,8 @@ variables_kw = (
     'xcoslib',
 )
 
-if __name__ == '__main__':
+
+if __name__ == '__main__':  # pragma: no cover
     import subprocess
     from pygments.util import format_lines, duplicates_removed
 

packages/wakatime/packages/pygments/lexers/_sourcemod_builtins.py

@@ -8,7 +8,7 @@
 
     Do not edit the FUNCTIONS list by hand.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -1091,7 +1091,9 @@ FUNCTIONS = (
     'SDKCall',
     'GetPlayerResourceEntity',
 )
-if __name__ == '__main__':
+
+
+if __name__ == '__main__':  # pragma: no cover
     import re
     import sys
     try:

packages/wakatime/packages/pygments/lexers/_stan_builtins.py

@@ -4,9 +4,9 @@
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     This file contains the names of functions for Stan used by
-    ``pygments.lexers.math.StanLexer. This is for Stan language version 2.4.0.
+    ``pygments.lexers.math.StanLexer. This is for Stan language version 2.8.0.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -16,10 +16,12 @@ KEYWORDS = (
     'if',
     'in',
     'increment_log_prob',
+    'integrate_ode',
     'lp__',
     'print',
+    'reject',
     'return',
-    'while',
+    'while'
 )
 
 TYPES = (
@@ -33,11 +35,11 @@ TYPES = (
     'positive_ordered',
     'real',
     'row_vector',
+    'row_vectormatrix',
     'simplex',
     'unit_vector',
     'vector',
-    'void',
-)
+    'void')
 
 FUNCTIONS = (
     'Phi',
@@ -45,6 +47,8 @@ FUNCTIONS = (
     'abs',
     'acos',
     'acosh',
+    'append_col',
+    'append_row',
     'asin',
     'asinh',
     'atan',
@@ -100,6 +104,11 @@ FUNCTIONS = (
     'cos',
     'cosh',
     'crossprod',
+    'csr_extract_u',
+    'csr_extract_v',
+    'csr_extract_w',
+    'csr_matrix_times_vector',
+    'csr_to_dense_matrix',
     'cumulative_sum',
     'determinant',
     'diag_matrix',
@@ -144,6 +153,11 @@ FUNCTIONS = (
     'fmax',
     'fmin',
     'fmod',
+    'frechet_ccdf_log',
+    'frechet_cdf',
+    'frechet_cdf_log',
+    'frechet_log',
+    'frechet_rng',
     'gamma_ccdf_log',
     'gamma_cdf',
     'gamma_cdf_log',
@@ -152,6 +166,7 @@ FUNCTIONS = (
     'gamma_q',
     'gamma_rng',
     'gaussian_dlm_obs_log',
+    'get_lp',
     'gumbel_ccdf_log',
     'gumbel_cdf',
     'gumbel_cdf_log',
@@ -176,19 +191,21 @@ FUNCTIONS = (
     'inv_gamma_log',
     'inv_gamma_rng',
     'inv_logit',
+    'inv_phi',
     'inv_sqrt',
     'inv_square',
     'inv_wishart_log',
     'inv_wishart_rng',
     'inverse',
     'inverse_spd',
+    'is_inf',
+    'is_nan',
     'lbeta',
     'lgamma',
     'lkj_corr_cholesky_log',
     'lkj_corr_cholesky_rng',
     'lkj_corr_log',
     'lkj_corr_rng',
-    'lkj_cov_log',
     'lmgamma',
     'log',
     'log10',
@@ -202,6 +219,7 @@ FUNCTIONS = (
     'log_diff_exp',
     'log_falling_factorial',
     'log_inv_logit',
+    'log_mix',
     'log_rising_factorial',
     'log_softmax',
     'log_sum_exp',
@@ -224,6 +242,7 @@ FUNCTIONS = (
     'min',
     'modified_bessel_first_kind',
     'modified_bessel_second_kind',
+    'multi_gp_cholesky_log',
     'multi_gp_log',
     'multi_normal_cholesky_log',
     'multi_normal_cholesky_rng',
@@ -236,6 +255,9 @@ FUNCTIONS = (
     'multinomial_rng',
     'multiply_log',
     'multiply_lower_tri_self_transpose',
+    'neg_binomial_2_ccdf_log',
+    'neg_binomial_2_cdf',
+    'neg_binomial_2_cdf_log',
     'neg_binomial_2_log',
     'neg_binomial_2_log_log',
     'neg_binomial_2_log_rng',
@@ -252,6 +274,7 @@ FUNCTIONS = (
     'normal_log',
     'normal_rng',
     'not_a_number',
+    'num_elements',
     'ordered_logistic_log',
     'ordered_logistic_rng',
     'owens_t',
@@ -260,12 +283,18 @@ FUNCTIONS = (
     'pareto_cdf_log',
     'pareto_log',
     'pareto_rng',
+    'pareto_type_2_ccdf_log',
+    'pareto_type_2_cdf',
+    'pareto_type_2_cdf_log',
+    'pareto_type_2_log',
+    'pareto_type_2_rng',
     'pi',
     'poisson_ccdf_log',
     'poisson_cdf',
     'poisson_cdf_log',
     'poisson_log',
     'poisson_log_log',
+    'poisson_log_rng',
     'poisson_rng',
     'positive_infinity',
     'pow',
@@ -353,8 +382,9 @@ FUNCTIONS = (
     'weibull_cdf_log',
     'weibull_log',
     'weibull_rng',
+    'wiener_log',
     'wishart_log',
-    'wishart_rng',
+    'wishart_rng'
 )
 
 DISTRIBUTIONS = (
@@ -372,6 +402,7 @@ DISTRIBUTIONS = (
     'double_exponential',
     'exp_mod_normal',
     'exponential',
+    'frechet',
     'gamma',
     'gaussian_dlm_obs',
     'gumbel',
@@ -381,10 +412,10 @@ DISTRIBUTIONS = (
     'inv_wishart',
     'lkj_corr',
     'lkj_corr_cholesky',
-    'lkj_cov',
     'logistic',
     'lognormal',
     'multi_gp',
+    'multi_gp_cholesky',
     'multi_normal',
     'multi_normal_cholesky',
     'multi_normal_prec',
@@ -396,6 +427,7 @@ DISTRIBUTIONS = (
     'normal',
     'ordered_logistic',
     'pareto',
+    'pareto_type_2',
     'poisson',
     'poisson_log',
     'rayleigh',
@@ -405,7 +437,8 @@ DISTRIBUTIONS = (
     'uniform',
     'von_mises',
     'weibull',
-    'wishart',
+    'wiener',
+    'wishart'
 )
 
 RESERVED = (
@@ -494,5 +527,6 @@ RESERVED = (
     'volatile',
     'wchar_t',
     'xor',
-    'xor_eq',
+    'xor_eq'
 )
+

packages/wakatime/packages/pygments/lexers/_stata_builtins.py

@@ -0,0 +1,419 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers._stata_builtins
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Builtins for Stata
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+
+builtins_base = (
+    "if", "else", "in", "foreach", "for", "forv", "forva",
+    "forval", "forvalu", "forvalue", "forvalues", "by", "bys",
+    "bysort", "quietly", "qui", "about", "ac",
+    "ac_7", "acprplot", "acprplot_7", "adjust", "ado", "adopath",
+    "adoupdate", "alpha", "ameans", "an", "ano", "anov", "anova",
+    "anova_estat", "anova_terms", "anovadef", "aorder", "ap", "app",
+    "appe", "appen", "append", "arch", "arch_dr", "arch_estat",
+    "arch_p", "archlm", "areg", "areg_p", "args", "arima",
+    "arima_dr", "arima_estat", "arima_p", "as", "asmprobit",
+    "asmprobit_estat", "asmprobit_lf", "asmprobit_mfx__dlg",
+    "asmprobit_p", "ass", "asse", "asser", "assert", "avplot",
+    "avplot_7", "avplots", "avplots_7", "bcskew0", "bgodfrey",
+    "binreg", "bip0_lf", "biplot", "bipp_lf", "bipr_lf",
+    "bipr_p", "biprobit", "bitest", "bitesti", "bitowt", "blogit",
+    "bmemsize", "boot", "bootsamp", "bootstrap", "bootstrap_8",
+    "boxco_l", "boxco_p", "boxcox", "boxcox_6", "boxcox_p",
+    "bprobit", "br", "break", "brier", "bro", "brow", "brows",
+    "browse", "brr", "brrstat", "bs", "bs_7", "bsampl_w",
+    "bsample", "bsample_7", "bsqreg", "bstat", "bstat_7", "bstat_8",
+    "bstrap", "bstrap_7", "ca", "ca_estat", "ca_p", "cabiplot",
+    "camat", "canon", "canon_8", "canon_8_p", "canon_estat",
+    "canon_p", "cap", "caprojection", "capt", "captu", "captur",
+    "capture", "cat", "cc", "cchart", "cchart_7", "cci",
+    "cd", "censobs_table", "centile", "cf", "char", "chdir",
+    "checkdlgfiles", "checkestimationsample", "checkhlpfiles",
+    "checksum", "chelp", "ci", "cii", "cl", "class", "classutil",
+    "clear", "cli", "clis", "clist", "clo", "clog", "clog_lf",
+    "clog_p", "clogi", "clogi_sw", "clogit", "clogit_lf",
+    "clogit_p", "clogitp", "clogl_sw", "cloglog", "clonevar",
+    "clslistarray", "cluster", "cluster_measures", "cluster_stop",
+    "cluster_tree", "cluster_tree_8", "clustermat", "cmdlog",
+    "cnr", "cnre", "cnreg", "cnreg_p", "cnreg_sw", "cnsreg",
+    "codebook", "collaps4", "collapse", "colormult_nb",
+    "colormult_nw", "compare", "compress", "conf", "confi",
+    "confir", "confirm", "conren", "cons", "const", "constr",
+    "constra", "constrai", "constrain", "constraint", "continue",
+    "contract", "copy", "copyright", "copysource", "cor", "corc",
+    "corr", "corr2data", "corr_anti", "corr_kmo", "corr_smc",
+    "corre", "correl", "correla", "correlat", "correlate",
+    "corrgram", "cou", "coun", "count", "cox", "cox_p", "cox_sw",
+    "coxbase", "coxhaz", "coxvar", "cprplot", "cprplot_7",
+    "crc", "cret", "cretu", "cretur", "creturn", "cross", "cs",
+    "cscript", "cscript_log", "csi", "ct", "ct_is", "ctset",
+    "ctst_5", "ctst_st", "cttost", "cumsp", "cumsp_7", "cumul",
+    "cusum", "cusum_7", "cutil", "d", "datasig", "datasign",
+    "datasigna", "datasignat", "datasignatu", "datasignatur",
+    "datasignature", "datetof", "db", "dbeta", "de", "dec",
+    "deco", "decod", "decode", "deff", "des", "desc", "descr",
+    "descri", "describ", "describe", "destring", "dfbeta",
+    "dfgls", "dfuller", "di", "di_g", "dir", "dirstats", "dis",
+    "discard", "disp", "disp_res", "disp_s", "displ", "displa",
+    "display", "distinct", "do", "doe", "doed", "doedi",
+    "doedit", "dotplot", "dotplot_7", "dprobit", "drawnorm",
+    "drop", "ds", "ds_util", "dstdize", "duplicates", "durbina",
+    "dwstat", "dydx", "e", "ed", "edi", "edit", "egen",
+    "eivreg", "emdef", "en", "enc", "enco", "encod", "encode",
+    "eq", "erase", "ereg", "ereg_lf", "ereg_p", "ereg_sw",
+    "ereghet", "ereghet_glf", "ereghet_glf_sh", "ereghet_gp",
+    "ereghet_ilf", "ereghet_ilf_sh", "ereghet_ip", "eret",
+    "eretu", "eretur", "ereturn", "err", "erro", "error", "est",
+    "est_cfexist", "est_cfname", "est_clickable", "est_expand",
+    "est_hold", "est_table", "est_unhold", "est_unholdok",
+    "estat", "estat_default", "estat_summ", "estat_vce_only",
+    "esti", "estimates", "etodow", "etof", "etomdy", "ex",
+    "exi", "exit", "expand", "expandcl", "fac", "fact", "facto",
+    "factor", "factor_estat", "factor_p", "factor_pca_rotated",
+    "factor_rotate", "factormat", "fcast", "fcast_compute",
+    "fcast_graph", "fdades", "fdadesc", "fdadescr", "fdadescri",
+    "fdadescrib", "fdadescribe", "fdasav", "fdasave", "fdause",
+    "fh_st", "open", "read", "close",
+    "file", "filefilter", "fillin", "find_hlp_file", "findfile",
+    "findit", "findit_7", "fit", "fl", "fli", "flis", "flist",
+    "for5_0", "form", "forma", "format", "fpredict", "frac_154",
+    "frac_adj", "frac_chk", "frac_cox", "frac_ddp", "frac_dis",
+    "frac_dv", "frac_in", "frac_mun", "frac_pp", "frac_pq",
+    "frac_pv", "frac_wgt", "frac_xo", "fracgen", "fracplot",
+    "fracplot_7", "fracpoly", "fracpred", "fron_ex", "fron_hn",
+    "fron_p", "fron_tn", "fron_tn2", "frontier", "ftodate", "ftoe",
+    "ftomdy", "ftowdate", "g", "gamhet_glf", "gamhet_gp",
+    "gamhet_ilf", "gamhet_ip", "gamma", "gamma_d2", "gamma_p",
+    "gamma_sw", "gammahet", "gdi_hexagon", "gdi_spokes", "ge",
+    "gen", "gene", "gener", "genera", "generat", "generate",
+    "genrank", "genstd", "genvmean", "gettoken", "gl", "gladder",
+    "gladder_7", "glim_l01", "glim_l02", "glim_l03", "glim_l04",
+    "glim_l05", "glim_l06", "glim_l07", "glim_l08", "glim_l09",
+    "glim_l10", "glim_l11", "glim_l12", "glim_lf", "glim_mu",
+    "glim_nw1", "glim_nw2", "glim_nw3", "glim_p", "glim_v1",
+    "glim_v2", "glim_v3", "glim_v4", "glim_v5", "glim_v6",
+    "glim_v7", "glm", "glm_6", "glm_p", "glm_sw", "glmpred", "glo",
+    "glob", "globa", "global", "glogit", "glogit_8", "glogit_p",
+    "gmeans", "gnbre_lf", "gnbreg", "gnbreg_5", "gnbreg_p",
+    "gomp_lf", "gompe_sw", "gomper_p", "gompertz", "gompertzhet",
+    "gomphet_glf", "gomphet_glf_sh", "gomphet_gp", "gomphet_ilf",
+    "gomphet_ilf_sh", "gomphet_ip", "gphdot", "gphpen",
+    "gphprint", "gprefs", "gprobi_p", "gprobit", "gprobit_8", "gr",
+    "gr7", "gr_copy", "gr_current", "gr_db", "gr_describe",
+    "gr_dir", "gr_draw", "gr_draw_replay", "gr_drop", "gr_edit",
+    "gr_editviewopts", "gr_example", "gr_example2", "gr_export",
+    "gr_print", "gr_qscheme", "gr_query", "gr_read", "gr_rename",
+    "gr_replay", "gr_save", "gr_set", "gr_setscheme", "gr_table",
+    "gr_undo", "gr_use", "graph", "graph7", "grebar", "greigen",
+    "greigen_7", "greigen_8", "grmeanby", "grmeanby_7",
+    "gs_fileinfo", "gs_filetype", "gs_graphinfo", "gs_stat",
+    "gsort", "gwood", "h", "hadimvo", "hareg", "hausman",
+    "haver", "he", "heck_d2", "heckma_p", "heckman", "heckp_lf",
+    "heckpr_p", "heckprob", "hel", "help", "hereg", "hetpr_lf",
+    "hetpr_p", "hetprob", "hettest", "hexdump", "hilite",
+    "hist", "hist_7", "histogram", "hlogit", "hlu", "hmeans",
+    "hotel", "hotelling", "hprobit", "hreg", "hsearch", "icd9",
+    "icd9_ff", "icd9p", "iis", "impute", "imtest", "inbase",
+    "include", "inf", "infi", "infil", "infile", "infix", "inp",
+    "inpu", "input", "ins", "insheet", "insp", "inspe",
+    "inspec", "inspect", "integ", "inten", "intreg", "intreg_7",
+    "intreg_p", "intrg2_ll", "intrg_ll", "intrg_ll2", "ipolate",
+    "iqreg", "ir", "irf", "irf_create", "irfm", "iri", "is_svy",
+    "is_svysum", "isid", "istdize", "ivprob_1_lf", "ivprob_lf",
+    "ivprobit", "ivprobit_p", "ivreg", "ivreg_footnote",
+    "ivtob_1_lf", "ivtob_lf", "ivtobit", "ivtobit_p", "jackknife",
+    "jacknife", "jknife", "jknife_6", "jknife_8", "jkstat",
+    "joinby", "kalarma1", "kap", "kap_3", "kapmeier", "kappa",
+    "kapwgt", "kdensity", "kdensity_7", "keep", "ksm", "ksmirnov",
+    "ktau", "kwallis", "l", "la", "lab", "labe", "label",
+    "labelbook", "ladder", "levels", "levelsof", "leverage",
+    "lfit", "lfit_p", "li", "lincom", "line", "linktest",
+    "lis", "list", "lloghet_glf", "lloghet_glf_sh", "lloghet_gp",
+    "lloghet_ilf", "lloghet_ilf_sh", "lloghet_ip", "llogi_sw",
+    "llogis_p", "llogist", "llogistic", "llogistichet",
+    "lnorm_lf", "lnorm_sw", "lnorma_p", "lnormal", "lnormalhet",
+    "lnormhet_glf", "lnormhet_glf_sh", "lnormhet_gp",
+    "lnormhet_ilf", "lnormhet_ilf_sh", "lnormhet_ip", "lnskew0",
+    "loadingplot", "loc", "loca", "local", "log", "logi",
+    "logis_lf", "logistic", "logistic_p", "logit", "logit_estat",
+    "logit_p", "loglogs", "logrank", "loneway", "lookfor",
+    "lookup", "lowess", "lowess_7", "lpredict", "lrecomp", "lroc",
+    "lroc_7", "lrtest", "ls", "lsens", "lsens_7", "lsens_x",
+    "lstat", "ltable", "ltable_7", "ltriang", "lv", "lvr2plot",
+    "lvr2plot_7", "m", "ma", "mac", "macr", "macro", "makecns",
+    "man", "manova", "manova_estat", "manova_p", "manovatest",
+    "mantel", "mark", "markin", "markout", "marksample", "mat",
+    "mat_capp", "mat_order", "mat_put_rr", "mat_rapp", "mata",
+    "mata_clear", "mata_describe", "mata_drop", "mata_matdescribe",
+    "mata_matsave", "mata_matuse", "mata_memory", "mata_mlib",
+    "mata_mosave", "mata_rename", "mata_which", "matalabel",
+    "matcproc", "matlist", "matname", "matr", "matri",
+    "matrix", "matrix_input__dlg", "matstrik", "mcc", "mcci",
+    "md0_", "md1_", "md1debug_", "md2_", "md2debug_", "mds",
+    "mds_estat", "mds_p", "mdsconfig", "mdslong", "mdsmat",
+    "mdsshepard", "mdytoe", "mdytof", "me_derd", "mean",
+    "means", "median", "memory", "memsize", "meqparse", "mer",
+    "merg", "merge", "mfp", "mfx", "mhelp", "mhodds", "minbound",
+    "mixed_ll", "mixed_ll_reparm", "mkassert", "mkdir",
+    "mkmat", "mkspline", "ml", "ml_5", "ml_adjs", "ml_bhhhs",
+    "ml_c_d", "ml_check", "ml_clear", "ml_cnt", "ml_debug",
+    "ml_defd", "ml_e0", "ml_e0_bfgs", "ml_e0_cycle", "ml_e0_dfp",
+    "ml_e0i", "ml_e1", "ml_e1_bfgs", "ml_e1_bhhh", "ml_e1_cycle",
+    "ml_e1_dfp", "ml_e2", "ml_e2_cycle", "ml_ebfg0", "ml_ebfr0",
+    "ml_ebfr1", "ml_ebh0q", "ml_ebhh0", "ml_ebhr0", "ml_ebr0i",
+    "ml_ecr0i", "ml_edfp0", "ml_edfr0", "ml_edfr1", "ml_edr0i",
+    "ml_eds", "ml_eer0i", "ml_egr0i", "ml_elf", "ml_elf_bfgs",
+    "ml_elf_bhhh", "ml_elf_cycle", "ml_elf_dfp", "ml_elfi",
+    "ml_elfs", "ml_enr0i", "ml_enrr0", "ml_erdu0", "ml_erdu0_bfgs",
+    "ml_erdu0_bhhh", "ml_erdu0_bhhhq", "ml_erdu0_cycle",
+    "ml_erdu0_dfp", "ml_erdu0_nrbfgs", "ml_exde", "ml_footnote",
+    "ml_geqnr", "ml_grad0", "ml_graph", "ml_hbhhh", "ml_hd0",
+    "ml_hold", "ml_init", "ml_inv", "ml_log", "ml_max",
+    "ml_mlout", "ml_mlout_8", "ml_model", "ml_nb0", "ml_opt",
+    "ml_p", "ml_plot", "ml_query", "ml_rdgrd", "ml_repor",
+    "ml_s_e", "ml_score", "ml_searc", "ml_technique", "ml_unhold",
+    "mleval", "mlf_", "mlmatbysum", "mlmatsum", "mlog", "mlogi",
+    "mlogit", "mlogit_footnote", "mlogit_p", "mlopts", "mlsum",
+    "mlvecsum", "mnl0_", "mor", "more", "mov", "move", "mprobit",
+    "mprobit_lf", "mprobit_p", "mrdu0_", "mrdu1_", "mvdecode",
+    "mvencode", "mvreg", "mvreg_estat", "n", "nbreg",
+    "nbreg_al", "nbreg_lf", "nbreg_p", "nbreg_sw", "nestreg", "net",
+    "newey", "newey_7", "newey_p", "news", "nl", "nl_7", "nl_9",
+    "nl_9_p", "nl_p", "nl_p_7", "nlcom", "nlcom_p", "nlexp2",
+    "nlexp2_7", "nlexp2a", "nlexp2a_7", "nlexp3", "nlexp3_7",
+    "nlgom3", "nlgom3_7", "nlgom4", "nlgom4_7", "nlinit", "nllog3",
+    "nllog3_7", "nllog4", "nllog4_7", "nlog_rd", "nlogit",
+    "nlogit_p", "nlogitgen", "nlogittree", "nlpred", "no",
+    "nobreak", "noi", "nois", "noisi", "noisil", "noisily", "note",
+    "notes", "notes_dlg", "nptrend", "numlabel", "numlist", "odbc",
+    "old_ver", "olo", "olog", "ologi", "ologi_sw", "ologit",
+    "ologit_p", "ologitp", "on", "one", "onew", "onewa", "oneway",
+    "op_colnm", "op_comp", "op_diff", "op_inv", "op_str", "opr",
+    "opro", "oprob", "oprob_sw", "oprobi", "oprobi_p", "oprobit",
+    "oprobitp", "opts_exclusive", "order", "orthog", "orthpoly",
+    "ou", "out", "outf", "outfi", "outfil", "outfile", "outs",
+    "outsh", "outshe", "outshee", "outsheet", "ovtest", "pac",
+    "pac_7", "palette", "parse", "parse_dissim", "pause", "pca",
+    "pca_8", "pca_display", "pca_estat", "pca_p", "pca_rotate",
+    "pcamat", "pchart", "pchart_7", "pchi", "pchi_7", "pcorr",
+    "pctile", "pentium", "pergram", "pergram_7", "permute",
+    "permute_8", "personal", "peto_st", "pkcollapse", "pkcross",
+    "pkequiv", "pkexamine", "pkexamine_7", "pkshape", "pksumm",
+    "pksumm_7", "pl", "plo", "plot", "plugin", "pnorm",
+    "pnorm_7", "poisgof", "poiss_lf", "poiss_sw", "poisso_p",
+    "poisson", "poisson_estat", "post", "postclose", "postfile",
+    "postutil", "pperron", "pr", "prais", "prais_e", "prais_e2",
+    "prais_p", "predict", "predictnl", "preserve", "print",
+    "pro", "prob", "probi", "probit", "probit_estat", "probit_p",
+    "proc_time", "procoverlay", "procrustes", "procrustes_estat",
+    "procrustes_p", "profiler", "prog", "progr", "progra",
+    "program", "prop", "proportion", "prtest", "prtesti", "pwcorr",
+    "pwd", "q", "s", "qby", "qbys", "qchi", "qchi_7", "qladder",
+    "qladder_7", "qnorm", "qnorm_7", "qqplot", "qqplot_7", "qreg",
+    "qreg_c", "qreg_p", "qreg_sw", "qu", "quadchk", "quantile",
+    "quantile_7", "que", "quer", "query", "range", "ranksum",
+    "ratio", "rchart", "rchart_7", "rcof", "recast", "reclink",
+    "recode", "reg", "reg3", "reg3_p", "regdw", "regr", "regre",
+    "regre_p2", "regres", "regres_p", "regress", "regress_estat",
+    "regriv_p", "remap", "ren", "rena", "renam", "rename",
+    "renpfix", "repeat", "replace", "report", "reshape",
+    "restore", "ret", "retu", "retur", "return", "rm", "rmdir",
+    "robvar", "roccomp", "roccomp_7", "roccomp_8", "rocf_lf",
+    "rocfit", "rocfit_8", "rocgold", "rocplot", "rocplot_7",
+    "roctab", "roctab_7", "rolling", "rologit", "rologit_p",
+    "rot", "rota", "rotat", "rotate", "rotatemat", "rreg",
+    "rreg_p", "ru", "run", "runtest", "rvfplot", "rvfplot_7",
+    "rvpplot", "rvpplot_7", "sa", "safesum", "sample",
+    "sampsi", "sav", "save", "savedresults", "saveold", "sc",
+    "sca", "scal", "scala", "scalar", "scatter", "scm_mine",
+    "sco", "scob_lf", "scob_p", "scobi_sw", "scobit", "scor",
+    "score", "scoreplot", "scoreplot_help", "scree", "screeplot",
+    "screeplot_help", "sdtest", "sdtesti", "se", "search",
+    "separate", "seperate", "serrbar", "serrbar_7", "serset", "set",
+    "set_defaults", "sfrancia", "sh", "she", "shel", "shell",
+    "shewhart", "shewhart_7", "signestimationsample", "signrank",
+    "signtest", "simul", "simul_7", "simulate", "simulate_8",
+    "sktest", "sleep", "slogit", "slogit_d2", "slogit_p", "smooth",
+    "snapspan", "so", "sor", "sort", "spearman", "spikeplot",
+    "spikeplot_7", "spikeplt", "spline_x", "split", "sqreg",
+    "sqreg_p", "sret", "sretu", "sretur", "sreturn", "ssc", "st",
+    "st_ct", "st_hc", "st_hcd", "st_hcd_sh", "st_is", "st_issys",
+    "st_note", "st_promo", "st_set", "st_show", "st_smpl",
+    "st_subid", "stack", "statsby", "statsby_8", "stbase", "stci",
+    "stci_7", "stcox", "stcox_estat", "stcox_fr", "stcox_fr_ll",
+    "stcox_p", "stcox_sw", "stcoxkm", "stcoxkm_7", "stcstat",
+    "stcurv", "stcurve", "stcurve_7", "stdes", "stem", "stepwise",
+    "stereg", "stfill", "stgen", "stir", "stjoin", "stmc", "stmh",
+    "stphplot", "stphplot_7", "stphtest", "stphtest_7",
+    "stptime", "strate", "strate_7", "streg", "streg_sw", "streset",
+    "sts", "sts_7", "stset", "stsplit", "stsum", "sttocc",
+    "sttoct", "stvary", "stweib", "su", "suest", "suest_8",
+    "sum", "summ", "summa", "summar", "summari", "summariz",
+    "summarize", "sunflower", "sureg", "survcurv", "survsum",
+    "svar", "svar_p", "svmat", "svy", "svy_disp", "svy_dreg",
+    "svy_est", "svy_est_7", "svy_estat", "svy_get", "svy_gnbreg_p",
+    "svy_head", "svy_header", "svy_heckman_p", "svy_heckprob_p",
+    "svy_intreg_p", "svy_ivreg_p", "svy_logistic_p", "svy_logit_p",
+    "svy_mlogit_p", "svy_nbreg_p", "svy_ologit_p", "svy_oprobit_p",
+    "svy_poisson_p", "svy_probit_p", "svy_regress_p", "svy_sub",
+    "svy_sub_7", "svy_x", "svy_x_7", "svy_x_p", "svydes",
+    "svydes_8", "svygen", "svygnbreg", "svyheckman", "svyheckprob",
+    "svyintreg", "svyintreg_7", "svyintrg", "svyivreg", "svylc",
+    "svylog_p", "svylogit", "svymarkout", "svymarkout_8",
+    "svymean", "svymlog", "svymlogit", "svynbreg", "svyolog",
+    "svyologit", "svyoprob", "svyoprobit", "svyopts",
+    "svypois", "svypois_7", "svypoisson", "svyprobit", "svyprobt",
+    "svyprop", "svyprop_7", "svyratio", "svyreg", "svyreg_p",
+    "svyregress", "svyset", "svyset_7", "svyset_8", "svytab",
+    "svytab_7", "svytest", "svytotal", "sw", "sw_8", "swcnreg",
+    "swcox", "swereg", "swilk", "swlogis", "swlogit",
+    "swologit", "swoprbt", "swpois", "swprobit", "swqreg",
+    "swtobit", "swweib", "symmetry", "symmi", "symplot",
+    "symplot_7", "syntax", "sysdescribe", "sysdir", "sysuse",
+    "szroeter", "ta", "tab", "tab1", "tab2", "tab_or", "tabd",
+    "tabdi", "tabdis", "tabdisp", "tabi", "table", "tabodds",
+    "tabodds_7", "tabstat", "tabu", "tabul", "tabula", "tabulat",
+    "tabulate", "te", "tempfile", "tempname", "tempvar", "tes",
+    "test", "testnl", "testparm", "teststd", "tetrachoric",
+    "time_it", "timer", "tis", "tob", "tobi", "tobit", "tobit_p",
+    "tobit_sw", "token", "tokeni", "tokeniz", "tokenize",
+    "tostring", "total", "translate", "translator", "transmap",
+    "treat_ll", "treatr_p", "treatreg", "trim", "trnb_cons",
+    "trnb_mean", "trpoiss_d2", "trunc_ll", "truncr_p", "truncreg",
+    "tsappend", "tset", "tsfill", "tsline", "tsline_ex",
+    "tsreport", "tsrevar", "tsrline", "tsset", "tssmooth",
+    "tsunab", "ttest", "ttesti", "tut_chk", "tut_wait", "tutorial",
+    "tw", "tware_st", "two", "twoway", "twoway__fpfit_serset",
+    "twoway__function_gen", "twoway__histogram_gen",
+    "twoway__ipoint_serset", "twoway__ipoints_serset",
+    "twoway__kdensity_gen", "twoway__lfit_serset",
+    "twoway__normgen_gen", "twoway__pci_serset",
+    "twoway__qfit_serset", "twoway__scatteri_serset",
+    "twoway__sunflower_gen", "twoway_ksm_serset", "ty", "typ",
+    "type", "typeof", "u", "unab", "unabbrev", "unabcmd",
+    "update", "us", "use", "uselabel", "var", "var_mkcompanion",
+    "var_p", "varbasic", "varfcast", "vargranger", "varirf",
+    "varirf_add", "varirf_cgraph", "varirf_create", "varirf_ctable",
+    "varirf_describe", "varirf_dir", "varirf_drop", "varirf_erase",
+    "varirf_graph", "varirf_ograph", "varirf_rename", "varirf_set",
+    "varirf_table", "varlist", "varlmar", "varnorm", "varsoc",
+    "varstable", "varstable_w", "varstable_w2", "varwle",
+    "vce", "vec", "vec_fevd", "vec_mkphi", "vec_p", "vec_p_w",
+    "vecirf_create", "veclmar", "veclmar_w", "vecnorm",
+    "vecnorm_w", "vecrank", "vecstable", "verinst", "vers",
+    "versi", "versio", "version", "view", "viewsource", "vif",
+    "vwls", "wdatetof", "webdescribe", "webseek", "webuse",
+    "weib1_lf", "weib2_lf", "weib_lf", "weib_lf0", "weibhet_glf",
+    "weibhet_glf_sh", "weibhet_glfa", "weibhet_glfa_sh",
+    "weibhet_gp", "weibhet_ilf", "weibhet_ilf_sh", "weibhet_ilfa",
+    "weibhet_ilfa_sh", "weibhet_ip", "weibu_sw", "weibul_p",
+    "weibull", "weibull_c", "weibull_s", "weibullhet",
+    "wh", "whelp", "whi", "which", "whil", "while", "wilc_st",
+    "wilcoxon", "win", "wind", "windo", "window", "winexec",
+    "wntestb", "wntestb_7", "wntestq", "xchart", "xchart_7",
+    "xcorr", "xcorr_7", "xi", "xi_6", "xmlsav", "xmlsave",
+    "xmluse", "xpose", "xsh", "xshe", "xshel", "xshell",
+    "xt_iis", "xt_tis", "xtab_p", "xtabond", "xtbin_p",
+    "xtclog", "xtcloglog", "xtcloglog_8", "xtcloglog_d2",
+    "xtcloglog_pa_p", "xtcloglog_re_p", "xtcnt_p", "xtcorr",
+    "xtdata", "xtdes", "xtfront_p", "xtfrontier", "xtgee",
+    "xtgee_elink", "xtgee_estat", "xtgee_makeivar", "xtgee_p",
+    "xtgee_plink", "xtgls", "xtgls_p", "xthaus", "xthausman",
+    "xtht_p", "xthtaylor", "xtile", "xtint_p", "xtintreg",
+    "xtintreg_8", "xtintreg_d2", "xtintreg_p", "xtivp_1",
+    "xtivp_2", "xtivreg", "xtline", "xtline_ex", "xtlogit",
+    "xtlogit_8", "xtlogit_d2", "xtlogit_fe_p", "xtlogit_pa_p",
+    "xtlogit_re_p", "xtmixed", "xtmixed_estat", "xtmixed_p",
+    "xtnb_fe", "xtnb_lf", "xtnbreg", "xtnbreg_pa_p",
+    "xtnbreg_refe_p", "xtpcse", "xtpcse_p", "xtpois", "xtpoisson",
+    "xtpoisson_d2", "xtpoisson_pa_p", "xtpoisson_refe_p", "xtpred",
+    "xtprobit", "xtprobit_8", "xtprobit_d2", "xtprobit_re_p",
+    "xtps_fe", "xtps_lf", "xtps_ren", "xtps_ren_8", "xtrar_p",
+    "xtrc", "xtrc_p", "xtrchh", "xtrefe_p", "xtreg", "xtreg_be",
+    "xtreg_fe", "xtreg_ml", "xtreg_pa_p", "xtreg_re",
+    "xtregar", "xtrere_p", "xtset", "xtsf_ll", "xtsf_llti",
+    "xtsum", "xttab", "xttest0", "xttobit", "xttobit_8",
+    "xttobit_p", "xttrans", "yx", "yxview__barlike_draw",
+    "yxview_area_draw", "yxview_bar_draw", "yxview_dot_draw",
+    "yxview_dropline_draw", "yxview_function_draw",
+    "yxview_iarrow_draw", "yxview_ilabels_draw",
+    "yxview_normal_draw", "yxview_pcarrow_draw",
+    "yxview_pcbarrow_draw", "yxview_pccapsym_draw",
+    "yxview_pcscatter_draw", "yxview_pcspike_draw",
+    "yxview_rarea_draw", "yxview_rbar_draw", "yxview_rbarm_draw",
+    "yxview_rcap_draw", "yxview_rcapsym_draw",
+    "yxview_rconnected_draw", "yxview_rline_draw",
+    "yxview_rscatter_draw", "yxview_rspike_draw",
+    "yxview_spike_draw", "yxview_sunflower_draw", "zap_s", "zinb",
+    "zinb_llf", "zinb_plf", "zip", "zip_llf", "zip_p", "zip_plf",
+    "zt_ct_5", "zt_hc_5", "zt_hcd_5", "zt_is_5", "zt_iss_5",
+    "zt_sho_5", "zt_smp_5", "ztbase_5", "ztcox_5", "ztdes_5",
+    "ztereg_5", "ztfill_5", "ztgen_5", "ztir_5", "ztjoin_5", "ztnb",
+    "ztnb_p", "ztp", "ztp_p", "zts_5", "ztset_5", "ztspli_5",
+    "ztsum_5", "zttoct_5", "ztvary_5", "ztweib_5"
+)
+
+builtins_functions = (
+    "Cdhms", "Chms", "Clock", "Cmdyhms", "Cofc", "Cofd", "F",
+    "Fden", "Ftail", "I", "J", "_caller", "abbrev", "abs", "acos",
+    "acosh", "asin", "asinh", "atan", "atan2", "atanh",
+    "autocode", "betaden", "binomial", "binomialp", "binomialtail",
+    "binormal", "bofd", "byteorder", "c", "ceil", "char",
+    "chi2", "chi2den", "chi2tail", "cholesky", "chop", "clip",
+    "clock", "cloglog", "cofC", "cofd", "colnumb", "colsof", "comb",
+    "cond", "corr", "cos", "cosh", "d", "daily", "date", "day",
+    "det", "dgammapda", "dgammapdada", "dgammapdadx", "dgammapdx",
+    "dgammapdxdx", "dhms", "diag", "diag0cnt", "digamma",
+    "dofC", "dofb", "dofc", "dofh", "dofm", "dofq", "dofw",
+    "dofy", "dow", "doy", "dunnettprob", "e", "el", "epsdouble",
+    "epsfloat", "exp", "fileexists", "fileread", "filereaderror",
+    "filewrite", "float", "floor", "fmtwidth", "gammaden",
+    "gammap", "gammaptail", "get", "group", "h", "hadamard",
+    "halfyear", "halfyearly", "has_eprop", "hh", "hhC", "hms",
+    "hofd", "hours", "hypergeometric", "hypergeometricp", "ibeta",
+    "ibetatail", "index", "indexnot", "inlist", "inrange", "int",
+    "inv", "invF", "invFtail", "invbinomial", "invbinomialtail",
+    "invchi2", "invchi2tail", "invcloglog", "invdunnettprob",
+    "invgammap", "invgammaptail", "invibeta", "invibetatail",
+    "invlogit", "invnFtail", "invnbinomial", "invnbinomialtail",
+    "invnchi2", "invnchi2tail", "invnibeta", "invnorm", "invnormal",
+    "invnttail", "invpoisson", "invpoissontail", "invsym", "invt",
+    "invttail", "invtukeyprob", "irecode", "issym", "issymmetric",
+    "itrim", "length", "ln", "lnfact", "lnfactorial", "lngamma",
+    "lnnormal", "lnnormalden", "log", "log10", "logit", "lower",
+    "ltrim", "m", "match", "matmissing", "matrix", "matuniform",
+    "max", "maxbyte", "maxdouble", "maxfloat", "maxint", "maxlong",
+    "mdy", "mdyhms", "mi", "min", "minbyte", "mindouble",
+    "minfloat", "minint", "minlong", "minutes", "missing", "mm",
+    "mmC", "mod", "mofd", "month", "monthly", "mreldif",
+    "msofhours", "msofminutes", "msofseconds", "nF", "nFden",
+    "nFtail", "nbetaden", "nbinomial", "nbinomialp", "nbinomialtail",
+    "nchi2", "nchi2den", "nchi2tail", "nibeta", "norm", "normal",
+    "normalden", "normd", "npnF", "npnchi2", "npnt", "nt", "ntden",
+    "nttail", "nullmat", "plural", "poisson", "poissonp",
+    "poissontail", "proper", "q", "qofd", "quarter", "quarterly",
+    "r", "rbeta", "rbinomial", "rchi2", "real", "recode", "regexm",
+    "regexr", "regexs", "reldif", "replay", "return", "reverse",
+    "rgamma", "rhypergeometric", "rnbinomial", "rnormal", "round",
+    "rownumb", "rowsof", "rpoisson", "rt", "rtrim", "runiform", "s",
+    "scalar", "seconds", "sign", "sin", "sinh", "smallestdouble",
+    "soundex", "soundex_nara", "sqrt", "ss", "ssC", "strcat",
+    "strdup", "string", "strlen", "strlower", "strltrim", "strmatch",
+    "strofreal", "strpos", "strproper", "strreverse", "strrtrim",
+    "strtoname", "strtrim", "strupper", "subinstr", "subinword",
+    "substr", "sum", "sweep", "syminv", "t", "tC", "tan", "tanh",
+    "tc", "td", "tden", "th", "tin", "tm", "tq", "trace",
+    "trigamma", "trim", "trunc", "ttail", "tukeyprob", "tw",
+    "twithin", "uniform", "upper", "vec", "vecdiag", "w", "week",
+    "weekly", "wofd", "word", "wordcount", "year", "yearly",
+    "yh", "ym", "yofd", "yq", "yw"
+)
+
+

packages/wakatime/packages/pygments/lexers/_vim_builtins.py

@@ -5,7 +5,7 @@
 
     This file is autogenerated by scripts/get_vimkw.py
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/actionscript.py

@@ -5,7 +5,7 @@
 
     Lexers for ActionScript and MXML.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/agile.py

@@ -5,7 +5,7 @@
 
     Just export lexer classes previously contained in this module.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/algebra.py

@@ -5,7 +5,7 @@
 
     Lexers for computer algebra systems.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -15,7 +15,7 @@ from pygments.lexer import RegexLexer, bygroups, words
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
     Number, Punctuation
 
-__all__ = ['GAPLexer', 'MathematicaLexer', 'MuPADLexer']
+__all__ = ['GAPLexer', 'MathematicaLexer', 'MuPADLexer', 'BCLexer']
 
 
 class GAPLexer(RegexLexer):
@@ -65,7 +65,7 @@ class GAPLexer(RegexLexer):
             (r'[0-9]+(?:\.[0-9]*)?(?:e[0-9]+)?', Number),
             (r'\.[0-9]+(?:e[0-9]+)?', Number),
             (r'.', Text)
-        ]
+        ],
     }
 
 
@@ -104,9 +104,9 @@ class MathematicaLexer(RegexLexer):
             (r'#\d*', Name.Variable),
             (r'([a-zA-Z]+[a-zA-Z0-9]*)', Name),
 
-            (r'-?[0-9]+\.[0-9]*', Number.Float),
-            (r'-?[0-9]*\.[0-9]+', Number.Float),
-            (r'-?[0-9]+', Number.Integer),
+            (r'-?\d+\.\d*', Number.Float),
+            (r'-?\d*\.\d+', Number.Float),
+            (r'-?\d+', Number.Integer),
 
             (words(operators), Operator),
             (words(punctuation), Punctuation),
@@ -183,5 +183,39 @@ class MuPADLexer(RegexLexer):
             (r'/\*', Comment.Multiline, '#push'),
             (r'\*/', Comment.Multiline, '#pop'),
             (r'[*/]', Comment.Multiline)
-        ]
+        ],
+    }
+
+
+class BCLexer(RegexLexer):
+    """
+    A `BC <https://www.gnu.org/software/bc/>`_ lexer.
+
+    .. versionadded:: 2.1
+    """
+    name = 'BC'
+    aliases = ['bc']
+    filenames = ['*.bc']
+
+    tokens = {
+        'root': [
+            (r'/\*', Comment.Multiline, 'comment'),
+            (r'"(?:[^"\\]|\\.)*"', String),
+            (r'[{}();,]', Punctuation),
+            (words(('if', 'else', 'while', 'for', 'break', 'continue',
+                    'halt', 'return', 'define', 'auto', 'print', 'read',
+                    'length', 'scale', 'sqrt', 'limits', 'quit',
+                    'warranty'), suffix=r'\b'), Keyword),
+            (r'\+\+|--|\|\||&&|'
+             r'([-<>+*%\^/!=])=?', Operator),
+            # bc doesn't support exponential
+            (r'[0-9]+(\.[0-9]*)?', Number),
+            (r'\.[0-9]+', Number),
+            (r'.', Text)
+        ],
+        'comment': [
+            (r'[^*/]+', Comment.Multiline),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'[*/]', Comment.Multiline)
+        ],
     }

packages/wakatime/packages/pygments/lexers/ambient.py

@@ -5,7 +5,7 @@
 
     Lexers for AmbientTalk language.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/ampl.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.ampl
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the ampl language. <http://ampl.com/>
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, using, this, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['AmplLexer']
+
+
+class AmplLexer(RegexLexer):
+    """
+    For AMPL source code.
+
+    .. versionadded:: 2.2
+    """
+    name = 'Ampl'
+    aliases = ['ampl']
+    filenames = ['*.run']
+
+    tokens = {
+        'root': [
+            (r'\n', Text),
+            (r'\s+', Text.Whitespace),
+            (r'#.*?\n', Comment.Single),
+            (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+            (words((
+                'call', 'cd', 'close', 'commands', 'data', 'delete', 'display',
+                'drop', 'end', 'environ', 'exit', 'expand', 'include', 'load',
+                'model', 'objective', 'option', 'problem', 'purge', 'quit',
+                'redeclare', 'reload', 'remove', 'reset', 'restore', 'shell',
+                'show', 'solexpand', 'solution', 'solve', 'update', 'unload',
+                'xref', 'coeff', 'coef', 'cover', 'obj', 'interval', 'default',
+                'from', 'to', 'to_come', 'net_in', 'net_out', 'dimen',
+                'dimension', 'check', 'complements', 'write', 'function',
+                'pipe', 'format', 'if', 'then', 'else', 'in', 'while', 'repeat',
+                'for'), suffix=r'\b'), Keyword.Reserved),
+            (r'(integer|binary|symbolic|ordered|circular|reversed|INOUT|IN|OUT|LOCAL)',
+             Keyword.Type),
+            (r'\".*?\"', String.Double),
+            (r'\'.*?\'', String.Single),
+            (r'[()\[\]{},;:]+', Punctuation),
+            (r'\b(\w+)(\.)(astatus|init0|init|lb0|lb1|lb2|lb|lrc|'
+             r'lslack|rc|relax|slack|sstatus|status|ub0|ub1|ub2|'
+             r'ub|urc|uslack|val)',
+             bygroups(Name.Variable, Punctuation, Keyword.Reserved)),
+            (r'(set|param|var|arc|minimize|maximize|subject to|s\.t\.|subj to|'
+             r'node|table|suffix|read table|write table)(\s+)(\w+)',
+             bygroups(Keyword.Declaration, Text, Name.Variable)),
+            (r'(param)(\s*)(:)(\s*)(\w+)(\s*)(:)(\s*)((\w|\s)+)',
+             bygroups(Keyword.Declaration, Text, Punctuation, Text,
+                      Name.Variable, Text, Punctuation, Text, Name.Variable)),
+            (r'(let|fix|unfix)(\s*)((?:\{.*\})?)(\s*)(\w+)',
+             bygroups(Keyword.Declaration, Text, using(this), Text, Name.Variable)),
+            (words((
+                'abs', 'acos', 'acosh', 'alias', 'asin', 'asinh', 'atan', 'atan2',
+                'atanh', 'ceil', 'ctime', 'cos', 'exp', 'floor', 'log', 'log10',
+                'max', 'min', 'precision', 'round', 'sin', 'sinh', 'sqrt', 'tan',
+                'tanh', 'time', 'trunc', 'Beta', 'Cauchy', 'Exponential', 'Gamma',
+                'Irand224', 'Normal', 'Normal01', 'Poisson', 'Uniform', 'Uniform01',
+                'num', 'num0', 'ichar', 'char', 'length', 'substr', 'sprintf',
+                'match', 'sub', 'gsub', 'print', 'printf', 'next', 'nextw', 'prev',
+                'prevw', 'first', 'last', 'ord', 'ord0', 'card', 'arity',
+                'indexarity'), prefix=r'\b', suffix=r'\b'), Name.Builtin),
+            (r'(\+|\-|\*|/|\*\*|=|<=|>=|==|\||\^|<|>|\!|\.\.|:=|\&|\!=|<<|>>)',
+             Operator),
+            (words((
+                'or', 'exists', 'forall', 'and', 'in', 'not', 'within', 'union',
+                'diff', 'difference', 'symdiff', 'inter', 'intersect',
+                'intersection', 'cross', 'setof', 'by', 'less', 'sum', 'prod',
+                'product', 'div', 'mod'), suffix=r'\b'),
+             Keyword.Reserved),  # Operator.Name but not enough emphasized with that
+            (r'(\d+\.(?!\.)\d*|\.(?!.)\d+)([eE][+-]?\d+)?', Number.Float),
+            (r'\d+([eE][+-]?\d+)?', Number.Integer),
+            (r'[+-]?Infinity', Number.Integer),
+            (r'(\w+|(\.(?!\.)))', Text)
+        ]
+
+    }

packages/wakatime/packages/pygments/lexers/apl.py

@@ -5,7 +5,7 @@
 
     Lexers for APL.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/archetype.py

@@ -0,0 +1,318 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.archetype
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Archetype-related syntaxes, including:
+
+    - ODIN syntax <https://github.com/openEHR/odin>
+    - ADL syntax <http://www.openehr.org/releases/trunk/architecture/am/adl2.pdf>
+    - cADL sub-syntax of ADL
+
+    For uses of this syntax, see the openEHR archetypes <http://www.openEHR.org/ckm>
+
+    Contributed by Thomas Beale <https://github.com/wolandscat>,
+    <https://bitbucket.org/thomas_beale>.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, bygroups, using, default
+from pygments.token import Text, Comment, Name, Literal, Number, String, \
+    Punctuation, Keyword, Operator, Generic
+
+__all__ = ['OdinLexer', 'CadlLexer', 'AdlLexer']
+
+
+class AtomsLexer(RegexLexer):
+    """
+    Lexer for Values used in ADL and ODIN.
+
+    .. versionadded:: 2.1
+    """
+
+    tokens = {
+        # ----- pseudo-states for inclusion -----
+        'whitespace': [
+            (r'\n', Text),
+            (r'\s+', Text),
+            (r'[ \t]*--.*$', Comment),
+        ],
+        'archetype_id': [
+            (r'[ \t]*([a-zA-Z]\w+(\.[a-zA-Z]\w+)*::)?[a-zA-Z]\w+(-[a-zA-Z]\w+){2}'
+             r'\.\w+[\w-]*\.v\d+(\.\d+){,2}((-[a-z]+)(\.\d+)?)?', Name.Decorator),
+        ],
+        'date_constraints': [
+            # ISO 8601-based date/time constraints
+            (r'[Xx?YyMmDdHhSs\d]{2,4}([:-][Xx?YyMmDdHhSs\d]{2}){2}', Literal.Date),
+            # ISO 8601-based duration constraints + optional trailing slash
+            (r'(P[YyMmWwDd]+(T[HhMmSs]+)?|PT[HhMmSs]+)/?', Literal.Date),
+        ],
+        'ordered_values': [
+            # ISO 8601 date with optional 'T' ligature
+            (r'\d{4}-\d{2}-\d{2}T?', Literal.Date),
+            # ISO 8601 time
+            (r'\d{2}:\d{2}:\d{2}(\.\d+)?([+-]\d{4}|Z)?', Literal.Date),
+            # ISO 8601 duration
+            (r'P((\d*(\.\d+)?[YyMmWwDd]){1,3}(T(\d*(\.\d+)?[HhMmSs]){,3})?|'
+             r'T(\d*(\.\d+)?[HhMmSs]){,3})', Literal.Date),
+            (r'[+-]?(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+', Number.Float),
+            (r'[+-]?(\d+)*\.\d+%?', Number.Float),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
+            (r'[+-]?\d+%?', Number.Integer),
+        ],
+        'values': [
+            include('ordered_values'),
+            (r'([Tt]rue|[Ff]alse)', Literal),
+            (r'"', String, 'string'),
+            (r"'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char),
+            (r'[a-z][a-z0-9+.-]*:', Literal, 'uri'),
+            # term code
+            (r'(\[)(\w[\w-]*(?:\([^)\n]+\))?)(::)(\w[\w-]*)(\])',
+             bygroups(Punctuation, Name.Decorator, Punctuation, Name.Decorator,
+                      Punctuation)),
+            (r'\|', Punctuation, 'interval'),
+            # list continuation
+            (r'\.\.\.', Punctuation),
+        ],
+        'constraint_values': [
+            (r'(\[)(\w[\w-]*(?:\([^)\n]+\))?)(::)',
+             bygroups(Punctuation, Name.Decorator, Punctuation), 'adl14_code_constraint'),
+            # ADL 1.4 ordinal constraint
+            (r'(\d*)(\|)(\[\w[\w-]*::\w[\w-]*\])((?:[,;])?)',
+             bygroups(Number, Punctuation, Name.Decorator, Punctuation)),
+            include('date_constraints'),
+            include('values'),
+        ],
+
+        # ----- real states -----
+        'string': [
+            ('"', String, '#pop'),
+            (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|'
+             r'u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|[0-7]{1,3})', String.Escape),
+            # all other characters
+            (r'[^\\"]+', String),
+            # stray backslash
+            (r'\\', String),
+        ],
+        'uri': [
+            # effective URI terminators
+            (r'[,>\s]', Punctuation, '#pop'),
+            (r'[^>\s,]+', Literal),
+        ],
+        'interval': [
+            (r'\|', Punctuation, '#pop'),
+            include('ordered_values'),
+            (r'\.\.', Punctuation),
+            (r'[<>=] *', Punctuation),
+            # handle +/-
+            (r'\+/-', Punctuation),
+            (r'\s+', Text),
+        ],
+        'any_code': [
+            include('archetype_id'),
+            # if it is a code
+            (r'[a-z_]\w*[0-9.]+(@[^\]]+)?', Name.Decorator),
+            # if it is tuple with attribute names
+            (r'[a-z_]\w*', Name.Class),
+            # if it is an integer, i.e. Xpath child index
+            (r'[0-9]+', Text),
+            (r'\|', Punctuation, 'code_rubric'),
+            (r'\]', Punctuation, '#pop'),
+            # handle use_archetype statement
+            (r'\s*,\s*', Punctuation),
+        ],
+        'code_rubric': [
+            (r'\|', Punctuation, '#pop'),
+            (r'[^|]+', String),
+        ],
+        'adl14_code_constraint': [
+            (r'\]', Punctuation, '#pop'),
+            (r'\|', Punctuation, 'code_rubric'),
+            (r'(\w[\w-]*)([;,]?)', bygroups(Name.Decorator, Punctuation)),
+            include('whitespace'),
+        ],
+    }
+
+
+class OdinLexer(AtomsLexer):
+    """
+    Lexer for ODIN syntax.
+
+    .. versionadded:: 2.1
+    """
+    name = 'ODIN'
+    aliases = ['odin']
+    filenames = ['*.odin']
+    mimetypes = ['text/odin']
+
+    tokens = {
+        'path': [
+            (r'>', Punctuation, '#pop'),
+            # attribute name
+            (r'[a-z_]\w*', Name.Class),
+            (r'/', Punctuation),
+            (r'\[', Punctuation, 'key'),
+            (r'\s*,\s*', Punctuation, '#pop'),
+            (r'\s+', Text, '#pop'),
+        ],
+        'key': [
+            include('values'),
+            (r'\]', Punctuation, '#pop'),
+        ],
+        'type_cast': [
+            (r'\)', Punctuation, '#pop'),
+            (r'[^)]+',  Name.Class),
+        ],
+        'root': [
+            include('whitespace'),
+            (r'([Tt]rue|[Ff]alse)', Literal),
+            include('values'),
+            # x-ref path
+            (r'/', Punctuation, 'path'),
+            # x-ref path starting with key
+            (r'\[', Punctuation, 'key'),
+            # attribute name
+            (r'[a-z_]\w*', Name.Class),
+            (r'=', Operator),
+            (r'\(', Punctuation, 'type_cast'),
+            (r',', Punctuation),
+            (r'<', Punctuation),
+            (r'>', Punctuation),
+            (r';', Punctuation),
+        ],
+    }
+
+
+class CadlLexer(AtomsLexer):
+    """
+    Lexer for cADL syntax.
+
+    .. versionadded:: 2.1
+    """
+    name = 'cADL'
+    aliases = ['cadl']
+    filenames = ['*.cadl']
+
+    tokens = {
+        'path': [
+            # attribute name
+            (r'[a-z_]\w*', Name.Class),
+            (r'/', Punctuation),
+            (r'\[', Punctuation, 'any_code'),
+            (r'\s+', Punctuation, '#pop'),
+        ],
+        'root': [
+            include('whitespace'),
+            (r'(cardinality|existence|occurrences|group|include|exclude|'
+             r'allow_archetype|use_archetype|use_node)\W', Keyword.Type),
+            (r'(and|or|not|there_exists|xor|implies|for_all)\W', Keyword.Type),
+            (r'(after|before|closed)\W', Keyword.Type),
+            (r'(not)\W', Operator),
+            (r'(matches|is_in)\W', Operator),
+            # is_in / not is_in char
+            (u'(\u2208|\u2209)', Operator),
+            # there_exists / not there_exists / for_all / and / or
+            (u'(\u2203|\u2204|\u2200|\u2227|\u2228|\u22BB|\223C)',
+             Operator),
+            # regex in slot or as string constraint
+            (r'(\{)(\s*/[^}]+/\s*)(\})',
+             bygroups(Punctuation, String.Regex, Punctuation)),
+            # regex in slot or as string constraint
+            (r'(\{)(\s*\^[^}]+\^\s*)(\})',
+             bygroups(Punctuation, String.Regex, Punctuation)),
+            (r'/', Punctuation, 'path'),
+            # for cardinality etc
+            (r'(\{)((?:\d+\.\.)?(?:\d+|\*))'
+             r'((?:\s*;\s*(?:ordered|unordered|unique)){,2})(\})',
+             bygroups(Punctuation, Number, Number, Punctuation)),
+            # [{ is start of a tuple value
+            (r'\[\{', Punctuation),
+            (r'\}\]', Punctuation),
+            (r'\{', Punctuation),
+            (r'\}', Punctuation),
+            include('constraint_values'),
+            # type name
+            (r'[A-Z]\w+(<[A-Z]\w+([A-Za-z_<>]*)>)?',  Name.Class),
+            # attribute name
+            (r'[a-z_]\w*', Name.Class),
+            (r'\[', Punctuation, 'any_code'),
+            (r'(~|//|\\\\|\+|-|/|\*|\^|!=|=|<=|>=|<|>]?)', Operator),
+            (r'\(', Punctuation),
+            (r'\)', Punctuation),
+            # for lists of values
+            (r',', Punctuation),
+            (r'"', String, 'string'),
+            # for assumed value
+            (r';', Punctuation),
+        ],
+    }
+
+
+class AdlLexer(AtomsLexer):
+    """
+    Lexer for ADL syntax.
+
+    .. versionadded:: 2.1
+    """
+
+    name = 'ADL'
+    aliases = ['adl']
+    filenames = ['*.adl', '*.adls', '*.adlf', '*.adlx']
+
+    tokens = {
+        'whitespace': [
+            # blank line ends
+            (r'\s*\n', Text),
+            # comment-only line
+            (r'^[ \t]*--.*$', Comment),
+        ],
+        'odin_section': [
+            # repeating the following two rules from the root state enable multi-line
+            # strings that start in the first column to be dealt with
+            (r'^(language|description|ontology|terminology|annotations|'
+             r'component_terminologies|revision_history)[ \t]*\n', Generic.Heading),
+            (r'^(definition)[ \t]*\n', Generic.Heading, 'cadl_section'),
+            (r'^([ \t]*|[ \t]+.*)\n', using(OdinLexer)),
+            (r'^([^"]*")(>[ \t]*\n)', bygroups(String, Punctuation)),
+            # template overlay delimiter
+            (r'^----------*\n', Text, '#pop'),
+            (r'^.*\n', String),
+            default('#pop'),
+        ],
+        'cadl_section': [
+            (r'^([ \t]*|[ \t]+.*)\n', using(CadlLexer)),
+            default('#pop'),
+        ],
+        'rules_section': [
+            (r'^[ \t]+.*\n', using(CadlLexer)),
+            default('#pop'),
+        ],
+        'metadata': [
+            (r'\)', Punctuation, '#pop'),
+            (r';', Punctuation),
+            (r'([Tt]rue|[Ff]alse)', Literal),
+            # numbers and version ids
+            (r'\d+(\.\d+)*', Literal),
+            # Guids
+            (r'(\d|[a-fA-F])+(-(\d|[a-fA-F])+){3,}', Literal),
+            (r'\w+', Name.Class),
+            (r'"', String, 'string'),
+            (r'=', Operator),
+            (r'[ \t]+', Text),
+            default('#pop'),
+        ],
+        'root': [
+            (r'^(archetype|template_overlay|operational_template|template|'
+             r'speciali[sz]e)', Generic.Heading),
+            (r'^(language|description|ontology|terminology|annotations|'
+             r'component_terminologies|revision_history)[ \t]*\n',
+             Generic.Heading, 'odin_section'),
+            (r'^(definition)[ \t]*\n', Generic.Heading, 'cadl_section'),
+            (r'^(rules)[ \t]*\n', Generic.Heading, 'rules_section'),
+            include('archetype_id'),
+            (r'[ \t]*\(', Punctuation, 'metadata'),
+            include('whitespace'),
+        ],
+    }

packages/wakatime/packages/pygments/lexers/asm.py

@@ -0,0 +1,641 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.asm
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for assembly languages.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using, words, \
+    DelegatingLexer
+from pygments.lexers.c_cpp import CppLexer, CLexer
+from pygments.lexers.d import DLexer
+from pygments.token import Text, Name, Number, String, Comment, Punctuation, \
+    Other, Keyword, Operator
+
+__all__ = ['GasLexer', 'ObjdumpLexer', 'DObjdumpLexer', 'CppObjdumpLexer',
+           'CObjdumpLexer', 'HsailLexer', 'LlvmLexer', 'NasmLexer',
+           'NasmObjdumpLexer', 'TasmLexer', 'Ca65Lexer']
+
+
+class GasLexer(RegexLexer):
+    """
+    For Gas (AT&T) assembly code.
+    """
+    name = 'GAS'
+    aliases = ['gas', 'asm']
+    filenames = ['*.s', '*.S']
+    mimetypes = ['text/x-gas']
+
+    #: optional Comment or Whitespace
+    string = r'"(\\"|[^"])*"'
+    char = r'[\w$.@-]'
+    identifier = r'(?:[a-zA-Z$_]' + char + '*|\.' + char + '+)'
+    number = r'(?:0[xX][a-zA-Z0-9]+|\d+)'
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            (identifier + ':', Name.Label),
+            (r'\.' + identifier, Name.Attribute, 'directive-args'),
+            (r'lock|rep(n?z)?|data\d+', Name.Attribute),
+            (identifier, Name.Function, 'instruction-args'),
+            (r'[\r\n]+', Text)
+        ],
+        'directive-args': [
+            (identifier, Name.Constant),
+            (string, String),
+            ('@' + identifier, Name.Attribute),
+            (number, Number.Integer),
+            (r'[\r\n]+', Text, '#pop'),
+
+            include('punctuation'),
+            include('whitespace')
+        ],
+        'instruction-args': [
+            # For objdump-disassembled code, shouldn't occur in
+            # actual assembler input
+            ('([a-z0-9]+)( )(<)('+identifier+')(>)',
+                bygroups(Number.Hex, Text, Punctuation, Name.Constant,
+                         Punctuation)),
+            ('([a-z0-9]+)( )(<)('+identifier+')([-+])('+number+')(>)',
+                bygroups(Number.Hex, Text, Punctuation, Name.Constant,
+                         Punctuation, Number.Integer, Punctuation)),
+
+            # Address constants
+            (identifier, Name.Constant),
+            (number, Number.Integer),
+            # Registers
+            ('%' + identifier, Name.Variable),
+            # Numeric constants
+            ('$'+number, Number.Integer),
+            (r"$'(.|\\')'", String.Char),
+            (r'[\r\n]+', Text, '#pop'),
+
+            include('punctuation'),
+            include('whitespace')
+        ],
+        'whitespace': [
+            (r'\n', Text),
+            (r'\s+', Text),
+            (r'[;#].*?\n', Comment)
+        ],
+        'punctuation': [
+            (r'[-*,.()\[\]!:]+', Punctuation)
+        ]
+    }
+
+    def analyse_text(text):
+        if re.match(r'^\.(text|data|section)', text, re.M):
+            return True
+        elif re.match(r'^\.\w+', text, re.M):
+            return 0.1
+
+
+def _objdump_lexer_tokens(asm_lexer):
+    """
+    Common objdump lexer tokens to wrap an ASM lexer.
+    """
+    hex_re = r'[0-9A-Za-z]'
+    return {
+        'root': [
+            # File name & format:
+            ('(.*?)(:)( +file format )(.*?)$',
+                bygroups(Name.Label, Punctuation, Text, String)),
+            # Section header
+            ('(Disassembly of section )(.*?)(:)$',
+                bygroups(Text, Name.Label, Punctuation)),
+            # Function labels
+            # (With offset)
+            ('('+hex_re+'+)( )(<)(.*?)([-+])(0[xX][A-Za-z0-9]+)(>:)$',
+                bygroups(Number.Hex, Text, Punctuation, Name.Function,
+                         Punctuation, Number.Hex, Punctuation)),
+            # (Without offset)
+            ('('+hex_re+'+)( )(<)(.*?)(>:)$',
+                bygroups(Number.Hex, Text, Punctuation, Name.Function,
+                         Punctuation)),
+            # Code line with disassembled instructions
+            ('( *)('+hex_re+r'+:)(\t)((?:'+hex_re+hex_re+' )+)( *\t)([a-zA-Z].*?)$',
+                bygroups(Text, Name.Label, Text, Number.Hex, Text,
+                         using(asm_lexer))),
+            # Code line with ascii
+            ('( *)('+hex_re+r'+:)(\t)((?:'+hex_re+hex_re+' )+)( *)(.*?)$',
+                bygroups(Text, Name.Label, Text, Number.Hex, Text, String)),
+            # Continued code line, only raw opcodes without disassembled
+            # instruction
+            ('( *)('+hex_re+r'+:)(\t)((?:'+hex_re+hex_re+' )+)$',
+                bygroups(Text, Name.Label, Text, Number.Hex)),
+            # Skipped a few bytes
+            (r'\t\.\.\.$', Text),
+            # Relocation line
+            # (With offset)
+            (r'(\t\t\t)('+hex_re+r'+:)( )([^\t]+)(\t)(.*?)([-+])(0x'+hex_re+'+)$',
+                bygroups(Text, Name.Label, Text, Name.Property, Text,
+                         Name.Constant, Punctuation, Number.Hex)),
+            # (Without offset)
+            (r'(\t\t\t)('+hex_re+r'+:)( )([^\t]+)(\t)(.*?)$',
+                bygroups(Text, Name.Label, Text, Name.Property, Text,
+                         Name.Constant)),
+            (r'[^\n]+\n', Other)
+        ]
+    }
+
+
+class ObjdumpLexer(RegexLexer):
+    """
+    For the output of 'objdump -dr'
+    """
+    name = 'objdump'
+    aliases = ['objdump']
+    filenames = ['*.objdump']
+    mimetypes = ['text/x-objdump']
+
+    tokens = _objdump_lexer_tokens(GasLexer)
+
+
+class DObjdumpLexer(DelegatingLexer):
+    """
+    For the output of 'objdump -Sr on compiled D files'
+    """
+    name = 'd-objdump'
+    aliases = ['d-objdump']
+    filenames = ['*.d-objdump']
+    mimetypes = ['text/x-d-objdump']
+
+    def __init__(self, **options):
+        super(DObjdumpLexer, self).__init__(DLexer, ObjdumpLexer, **options)
+
+
+class CppObjdumpLexer(DelegatingLexer):
+    """
+    For the output of 'objdump -Sr on compiled C++ files'
+    """
+    name = 'cpp-objdump'
+    aliases = ['cpp-objdump', 'c++-objdumb', 'cxx-objdump']
+    filenames = ['*.cpp-objdump', '*.c++-objdump', '*.cxx-objdump']
+    mimetypes = ['text/x-cpp-objdump']
+
+    def __init__(self, **options):
+        super(CppObjdumpLexer, self).__init__(CppLexer, ObjdumpLexer, **options)
+
+
+class CObjdumpLexer(DelegatingLexer):
+    """
+    For the output of 'objdump -Sr on compiled C files'
+    """
+    name = 'c-objdump'
+    aliases = ['c-objdump']
+    filenames = ['*.c-objdump']
+    mimetypes = ['text/x-c-objdump']
+
+    def __init__(self, **options):
+        super(CObjdumpLexer, self).__init__(CLexer, ObjdumpLexer, **options)
+
+
+class HsailLexer(RegexLexer):
+    """
+    For HSAIL assembly code.
+
+    .. versionadded:: 2.2
+    """
+    name = 'HSAIL'
+    aliases = ['hsail', 'hsa']
+    filenames = ['*.hsail']
+    mimetypes = ['text/x-hsail']
+
+    string = r'"[^"]*?"'
+    identifier = r'[a-zA-Z_][\w.]*'
+    # Registers
+    register_number = r'[0-9]+'
+    register = r'(\$(c|s|d|q)' + register_number + ')'
+    # Qualifiers
+    alignQual = r'(align\(\d+\))'
+    widthQual = r'(width\((\d+|all)\))'
+    allocQual = r'(alloc\(agent\))'
+    # Instruction Modifiers
+    roundingMod = (r'((_ftz)?(_up|_down|_zero|_near))')
+    datatypeMod = (r'_('
+                   # packedTypes
+                   r'u8x4|s8x4|u16x2|s16x2|u8x8|s8x8|u16x4|s16x4|u32x2|s32x2|'
+                   r'u8x16|s8x16|u16x8|s16x8|u32x4|s32x4|u64x2|s64x2|'
+                   r'f16x2|f16x4|f16x8|f32x2|f32x4|f64x2|'
+                   # baseTypes
+                   r'u8|s8|u16|s16|u32|s32|u64|s64|'
+                   r'b128|b8|b16|b32|b64|b1|'
+                   r'f16|f32|f64|'
+                   # opaqueType
+                   r'roimg|woimg|rwimg|samp|sig32|sig64)')
+
+    # Numeric Constant
+    float = r'((\d+\.)|(\d*\.\d+))[eE][+-]?\d+'
+    hexfloat = r'0[xX](([0-9a-fA-F]+\.[0-9a-fA-F]*)|([0-9a-fA-F]*\.[0-9a-fA-F]+))[pP][+-]?\d+'
+    ieeefloat = r'0((h|H)[0-9a-fA-F]{4}|(f|F)[0-9a-fA-F]{8}|(d|D)[0-9a-fA-F]{16})'
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            include('comments'),
+
+            (string, String),
+
+            (r'@' + identifier + ':?', Name.Label),
+
+            (register, Name.Variable.Anonymous),
+
+            include('keyword'),
+
+            (r'&' + identifier, Name.Variable.Global),
+            (r'%' + identifier, Name.Variable),
+
+            (hexfloat, Number.Hex),
+            (r'0[xX][a-fA-F0-9]+', Number.Hex),
+            (ieeefloat, Number.Float),
+            (float, Number.Float),
+            ('\d+', Number.Integer),
+
+            (r'[=<>{}\[\]()*.,:;!]|x\b', Punctuation)
+        ],
+        'whitespace': [
+            (r'(\n|\s)+', Text),
+        ],
+        'comments': [
+            (r'/\*.*?\*/', Comment.Multiline),
+            (r'//.*?\n', Comment.Single),
+        ],
+        'keyword': [
+            # Types
+            (r'kernarg' + datatypeMod, Keyword.Type),
+
+            # Regular keywords
+            (r'\$(full|base|small|large|default|zero|near)', Keyword),
+            (words((
+                'module', 'extension', 'pragma', 'prog', 'indirect', 'signature',
+                'decl', 'kernel', 'function', 'enablebreakexceptions',
+                'enabledetectexceptions', 'maxdynamicgroupsize', 'maxflatgridsize',
+                'maxflatworkgroupsize', 'requireddim', 'requiredgridsize',
+                'requiredworkgroupsize', 'requirenopartialworkgroups'),
+                suffix=r'\b'), Keyword),
+
+            # instructions
+            (roundingMod, Keyword),
+            (datatypeMod, Keyword),
+            (r'_(' + alignQual + '|' + widthQual + ')', Keyword),
+            (r'_kernarg', Keyword),
+            (r'(nop|imagefence)\b', Keyword),
+            (words((
+                'cleardetectexcept', 'clock', 'cuid', 'debugtrap', 'dim',
+                'getdetectexcept', 'groupbaseptr', 'kernargbaseptr', 'laneid',
+                'maxcuid', 'maxwaveid', 'packetid', 'setdetectexcept', 'waveid',
+                'workitemflatabsid', 'workitemflatid', 'nullptr', 'abs', 'bitrev',
+                'currentworkgroupsize', 'currentworkitemflatid', 'fract', 'ncos',
+                'neg', 'nexp2', 'nlog2', 'nrcp', 'nrsqrt', 'nsin', 'nsqrt',
+                'gridgroups', 'gridsize', 'not', 'sqrt', 'workgroupid',
+                'workgroupsize', 'workitemabsid', 'workitemid', 'ceil', 'floor',
+                'rint', 'trunc', 'add', 'bitmask', 'borrow', 'carry', 'copysign',
+                'div', 'rem', 'sub', 'shl', 'shr', 'and', 'or', 'xor', 'unpackhi',
+                'unpacklo', 'max', 'min', 'fma', 'mad', 'bitextract', 'bitselect',
+                'shuffle', 'cmov', 'bitalign', 'bytealign', 'lerp', 'nfma', 'mul',
+                'mulhi', 'mul24hi', 'mul24', 'mad24', 'mad24hi', 'bitinsert',
+                'combine', 'expand', 'lda', 'mov', 'pack', 'unpack', 'packcvt',
+                'unpackcvt', 'sad', 'sementp', 'ftos', 'stof', 'cmp', 'ld', 'st',
+                '_eq', '_ne', '_lt', '_le', '_gt', '_ge', '_equ', '_neu', '_ltu',
+                '_leu', '_gtu', '_geu', '_num', '_nan', '_seq', '_sne', '_slt',
+                '_sle', '_sgt', '_sge', '_snum', '_snan', '_sequ', '_sneu', '_sltu',
+                '_sleu', '_sgtu', '_sgeu', 'atomic', '_ld', '_st', '_cas', '_add',
+                '_and', '_exch', '_max', '_min', '_or', '_sub', '_wrapdec',
+                '_wrapinc', '_xor', 'ret', 'cvt', '_readonly', '_kernarg', '_global',
+                'br', 'cbr', 'sbr', '_scacq', '_screl', '_scar', '_rlx', '_wave',
+                '_wg', '_agent', '_system', 'ldimage', 'stimage', '_v2', '_v3', '_v4',
+                '_1d', '_2d', '_3d', '_1da', '_2da', '_1db', '_2ddepth', '_2dadepth',
+                '_width', '_height', '_depth', '_array', '_channelorder',
+                '_channeltype', 'querysampler', '_coord', '_filter', '_addressing',
+                'barrier', 'wavebarrier', 'initfbar', 'joinfbar', 'waitfbar',
+                'arrivefbar', 'leavefbar', 'releasefbar', 'ldf', 'activelaneid',
+                'activelanecount', 'activelanemask', 'activelanepermute', 'call',
+                'scall', 'icall', 'alloca', 'packetcompletionsig',
+                'addqueuewriteindex', 'casqueuewriteindex', 'ldqueuereadindex',
+                'stqueuereadindex', 'readonly', 'global', 'private', 'group',
+                'spill', 'arg', '_upi', '_downi', '_zeroi', '_neari', '_upi_sat',
+                '_downi_sat', '_zeroi_sat', '_neari_sat', '_supi', '_sdowni',
+                '_szeroi', '_sneari', '_supi_sat', '_sdowni_sat', '_szeroi_sat',
+                '_sneari_sat', '_pp', '_ps', '_sp', '_ss', '_s', '_p', '_pp_sat',
+                '_ps_sat', '_sp_sat', '_ss_sat', '_s_sat', '_p_sat')), Keyword),
+
+            # Integer types
+            (r'i[1-9]\d*', Keyword)
+        ]
+    }
+
+
+class LlvmLexer(RegexLexer):
+    """
+    For LLVM assembly code.
+    """
+    name = 'LLVM'
+    aliases = ['llvm']
+    filenames = ['*.ll']
+    mimetypes = ['text/x-llvm']
+
+    #: optional Comment or Whitespace
+    string = r'"[^"]*?"'
+    identifier = r'([-a-zA-Z$._][\w\-$.]*|' + string + ')'
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+
+            # Before keywords, because keywords are valid label names :(...
+            (identifier + '\s*:', Name.Label),
+
+            include('keyword'),
+
+            (r'%' + identifier, Name.Variable),
+            (r'@' + identifier, Name.Variable.Global),
+            (r'%\d+', Name.Variable.Anonymous),
+            (r'@\d+', Name.Variable.Global),
+            (r'#\d+', Name.Variable.Global),
+            (r'!' + identifier, Name.Variable),
+            (r'!\d+', Name.Variable.Anonymous),
+            (r'c?' + string, String),
+
+            (r'0[xX][a-fA-F0-9]+', Number),
+            (r'-?\d+(?:[.]\d+)?(?:[eE][-+]?\d+(?:[.]\d+)?)?', Number),
+
+            (r'[=<>{}\[\]()*.,!]|x\b', Punctuation)
+        ],
+        'whitespace': [
+            (r'(\n|\s)+', Text),
+            (r';.*?\n', Comment)
+        ],
+        'keyword': [
+            # Regular keywords
+            (words((
+                'begin', 'end', 'true', 'false', 'declare', 'define', 'global',
+                'constant', 'private', 'linker_private', 'internal',
+                'available_externally', 'linkonce', 'linkonce_odr', 'weak',
+                'weak_odr', 'appending', 'dllimport', 'dllexport', 'common',
+                'default', 'hidden', 'protected', 'extern_weak', 'external',
+                'thread_local', 'zeroinitializer', 'undef', 'null', 'to', 'tail',
+                'target', 'triple', 'datalayout', 'volatile', 'nuw', 'nsw', 'nnan',
+                'ninf', 'nsz', 'arcp', 'fast', 'exact', 'inbounds', 'align',
+                'addrspace', 'section', 'alias', 'module', 'asm', 'sideeffect',
+                'gc', 'dbg', 'linker_private_weak', 'attributes', 'blockaddress',
+                'initialexec', 'localdynamic', 'localexec', 'prefix', 'unnamed_addr',
+                'ccc', 'fastcc', 'coldcc', 'x86_stdcallcc', 'x86_fastcallcc',
+                'arm_apcscc', 'arm_aapcscc', 'arm_aapcs_vfpcc', 'ptx_device',
+                'ptx_kernel', 'intel_ocl_bicc', 'msp430_intrcc', 'spir_func',
+                'spir_kernel', 'x86_64_sysvcc', 'x86_64_win64cc', 'x86_thiscallcc',
+                'cc', 'c', 'signext', 'zeroext', 'inreg', 'sret', 'nounwind',
+                'noreturn', 'noalias', 'nocapture', 'byval', 'nest', 'readnone',
+                'readonly', 'inlinehint', 'noinline', 'alwaysinline', 'optsize', 'ssp',
+                'sspreq', 'noredzone', 'noimplicitfloat', 'naked', 'builtin', 'cold',
+                'nobuiltin', 'noduplicate', 'nonlazybind', 'optnone', 'returns_twice',
+                'sanitize_address', 'sanitize_memory', 'sanitize_thread', 'sspstrong',
+                'uwtable', 'returned', 'type', 'opaque', 'eq', 'ne', 'slt', 'sgt',
+                'sle', 'sge', 'ult', 'ugt', 'ule', 'uge', 'oeq', 'one', 'olt', 'ogt',
+                'ole', 'oge', 'ord', 'uno', 'ueq', 'une', 'x', 'acq_rel', 'acquire',
+                'alignstack', 'atomic', 'catch', 'cleanup', 'filter', 'inteldialect',
+                'max', 'min', 'monotonic', 'nand', 'personality', 'release', 'seq_cst',
+                'singlethread', 'umax', 'umin', 'unordered', 'xchg', 'add', 'fadd',
+                'sub', 'fsub', 'mul', 'fmul', 'udiv', 'sdiv', 'fdiv', 'urem', 'srem',
+                'frem', 'shl', 'lshr', 'ashr', 'and', 'or', 'xor', 'icmp', 'fcmp',
+                'phi', 'call', 'trunc', 'zext', 'sext', 'fptrunc', 'fpext', 'uitofp',
+                'sitofp', 'fptoui', 'fptosi', 'inttoptr', 'ptrtoint', 'bitcast',
+                'addrspacecast', 'select', 'va_arg', 'ret', 'br', 'switch', 'invoke',
+                'unwind', 'unreachable', 'indirectbr', 'landingpad', 'resume',
+                'malloc', 'alloca', 'free', 'load', 'store', 'getelementptr',
+                'extractelement', 'insertelement', 'shufflevector', 'getresult',
+                'extractvalue', 'insertvalue', 'atomicrmw', 'cmpxchg', 'fence',
+                'allocsize', 'amdgpu_cs', 'amdgpu_gs', 'amdgpu_kernel', 'amdgpu_ps',
+                'amdgpu_vs', 'any', 'anyregcc', 'argmemonly', 'avr_intrcc',
+                'avr_signalcc', 'caller', 'catchpad', 'catchret', 'catchswitch',
+                'cleanuppad', 'cleanupret', 'comdat', 'convergent', 'cxx_fast_tlscc',
+                'deplibs', 'dereferenceable', 'dereferenceable_or_null', 'distinct',
+                'exactmatch', 'externally_initialized', 'from', 'ghccc', 'hhvm_ccc',
+                'hhvmcc', 'ifunc', 'inaccessiblemem_or_argmemonly', 'inaccessiblememonly',
+                'inalloca', 'jumptable', 'largest', 'local_unnamed_addr', 'minsize',
+                'musttail', 'noduplicates', 'none', 'nonnull', 'norecurse', 'notail',
+                'preserve_allcc', 'preserve_mostcc', 'prologue', 'safestack', 'samesize',
+                'source_filename', 'swiftcc', 'swifterror', 'swiftself', 'webkit_jscc',
+                'within', 'writeonly', 'x86_intrcc', 'x86_vectorcallcc'),
+                suffix=r'\b'), Keyword),
+
+            # Types
+            (words(('void', 'half', 'float', 'double', 'x86_fp80', 'fp128',
+                    'ppc_fp128', 'label', 'metadata', 'token')), Keyword.Type),
+
+            # Integer types
+            (r'i[1-9]\d*', Keyword)
+        ]
+    }
+
+
+class NasmLexer(RegexLexer):
+    """
+    For Nasm (Intel) assembly code.
+    """
+    name = 'NASM'
+    aliases = ['nasm']
+    filenames = ['*.asm', '*.ASM']
+    mimetypes = ['text/x-nasm']
+
+    identifier = r'[a-z$._?][\w$.?#@~]*'
+    hexn = r'(?:0x[0-9a-f]+|$0[0-9a-f]*|[0-9]+[0-9a-f]*h)'
+    octn = r'[0-7]+q'
+    binn = r'[01]+b'
+    decn = r'[0-9]+'
+    floatn = decn + r'\.e?' + decn
+    string = r'"(\\"|[^"\n])*"|' + r"'(\\'|[^'\n])*'|" + r"`(\\`|[^`\n])*`"
+    declkw = r'(?:res|d)[bwdqt]|times'
+    register = (r'r[0-9][0-5]?[bwd]|'
+                r'[a-d][lh]|[er]?[a-d]x|[er]?[sb]p|[er]?[sd]i|[c-gs]s|st[0-7]|'
+                r'mm[0-7]|cr[0-4]|dr[0-367]|tr[3-7]')
+    wordop = r'seg|wrt|strict'
+    type = r'byte|[dq]?word'
+    directives = (r'BITS|USE16|USE32|SECTION|SEGMENT|ABSOLUTE|EXTERN|GLOBAL|'
+                  r'ORG|ALIGN|STRUC|ENDSTRUC|COMMON|CPU|GROUP|UPPERCASE|IMPORT|'
+                  r'EXPORT|LIBRARY|MODULE')
+
+    flags = re.IGNORECASE | re.MULTILINE
+    tokens = {
+        'root': [
+            (r'^\s*%', Comment.Preproc, 'preproc'),
+            include('whitespace'),
+            (identifier + ':', Name.Label),
+            (r'(%s)(\s+)(equ)' % identifier,
+                bygroups(Name.Constant, Keyword.Declaration, Keyword.Declaration),
+                'instruction-args'),
+            (directives, Keyword, 'instruction-args'),
+            (declkw, Keyword.Declaration, 'instruction-args'),
+            (identifier, Name.Function, 'instruction-args'),
+            (r'[\r\n]+', Text)
+        ],
+        'instruction-args': [
+            (string, String),
+            (hexn, Number.Hex),
+            (octn, Number.Oct),
+            (binn, Number.Bin),
+            (floatn, Number.Float),
+            (decn, Number.Integer),
+            include('punctuation'),
+            (register, Name.Builtin),
+            (identifier, Name.Variable),
+            (r'[\r\n]+', Text, '#pop'),
+            include('whitespace')
+        ],
+        'preproc': [
+            (r'[^;\n]+', Comment.Preproc),
+            (r';.*?\n', Comment.Single, '#pop'),
+            (r'\n', Comment.Preproc, '#pop'),
+        ],
+        'whitespace': [
+            (r'\n', Text),
+            (r'[ \t]+', Text),
+            (r';.*', Comment.Single)
+        ],
+        'punctuation': [
+            (r'[,():\[\]]+', Punctuation),
+            (r'[&|^<>+*/%~-]+', Operator),
+            (r'[$]+', Keyword.Constant),
+            (wordop, Operator.Word),
+            (type, Keyword.Type)
+        ],
+    }
+
+
+class NasmObjdumpLexer(ObjdumpLexer):
+    """
+    For the output of 'objdump -d -M intel'.
+
+    .. versionadded:: 2.0
+    """
+    name = 'objdump-nasm'
+    aliases = ['objdump-nasm']
+    filenames = ['*.objdump-intel']
+    mimetypes = ['text/x-nasm-objdump']
+
+    tokens = _objdump_lexer_tokens(NasmLexer)
+
+
+class TasmLexer(RegexLexer):
+    """
+    For Tasm (Turbo Assembler) assembly code.
+    """
+    name = 'TASM'
+    aliases = ['tasm']
+    filenames = ['*.asm', '*.ASM', '*.tasm']
+    mimetypes = ['text/x-tasm']
+
+    identifier = r'[@a-z$._?][\w$.?#@~]*'
+    hexn = r'(?:0x[0-9a-f]+|$0[0-9a-f]*|[0-9]+[0-9a-f]*h)'
+    octn = r'[0-7]+q'
+    binn = r'[01]+b'
+    decn = r'[0-9]+'
+    floatn = decn + r'\.e?' + decn
+    string = r'"(\\"|[^"\n])*"|' + r"'(\\'|[^'\n])*'|" + r"`(\\`|[^`\n])*`"
+    declkw = r'(?:res|d)[bwdqt]|times'
+    register = (r'r[0-9][0-5]?[bwd]|'
+                r'[a-d][lh]|[er]?[a-d]x|[er]?[sb]p|[er]?[sd]i|[c-gs]s|st[0-7]|'
+                r'mm[0-7]|cr[0-4]|dr[0-367]|tr[3-7]')
+    wordop = r'seg|wrt|strict'
+    type = r'byte|[dq]?word'
+    directives = (r'BITS|USE16|USE32|SECTION|SEGMENT|ABSOLUTE|EXTERN|GLOBAL|'
+                  r'ORG|ALIGN|STRUC|ENDSTRUC|ENDS|COMMON|CPU|GROUP|UPPERCASE|INCLUDE|'
+                  r'EXPORT|LIBRARY|MODULE|PROC|ENDP|USES|ARG|DATASEG|UDATASEG|END|IDEAL|'
+                  r'P386|MODEL|ASSUME|CODESEG|SIZE')
+    # T[A-Z][a-z] is more of a convention. Lexer should filter out STRUC definitions
+    # and then 'add' them to datatype somehow.
+    datatype = (r'db|dd|dw|T[A-Z][a-z]+')
+
+    flags = re.IGNORECASE | re.MULTILINE
+    tokens = {
+        'root': [
+            (r'^\s*%', Comment.Preproc, 'preproc'),
+            include('whitespace'),
+            (identifier + ':', Name.Label),
+            (directives, Keyword, 'instruction-args'),
+            (r'(%s)(\s+)(%s)' % (identifier, datatype),
+                bygroups(Name.Constant, Keyword.Declaration, Keyword.Declaration),
+                'instruction-args'),
+            (declkw, Keyword.Declaration, 'instruction-args'),
+            (identifier, Name.Function, 'instruction-args'),
+            (r'[\r\n]+', Text)
+        ],
+        'instruction-args': [
+            (string, String),
+            (hexn, Number.Hex),
+            (octn, Number.Oct),
+            (binn, Number.Bin),
+            (floatn, Number.Float),
+            (decn, Number.Integer),
+            include('punctuation'),
+            (register, Name.Builtin),
+            (identifier, Name.Variable),
+            # Do not match newline when it's preceeded by a backslash
+            (r'(\\\s*)(;.*)([\r\n])', bygroups(Text, Comment.Single, Text)),
+            (r'[\r\n]+', Text, '#pop'),
+            include('whitespace')
+        ],
+        'preproc': [
+            (r'[^;\n]+', Comment.Preproc),
+            (r';.*?\n', Comment.Single, '#pop'),
+            (r'\n', Comment.Preproc, '#pop'),
+        ],
+        'whitespace': [
+            (r'[\n\r]', Text),
+            (r'\\[\n\r]', Text),
+            (r'[ \t]+', Text),
+            (r';.*', Comment.Single)
+        ],
+        'punctuation': [
+            (r'[,():\[\]]+', Punctuation),
+            (r'[&|^<>+*=/%~-]+', Operator),
+            (r'[$]+', Keyword.Constant),
+            (wordop, Operator.Word),
+            (type, Keyword.Type)
+        ],
+    }
+
+
+class Ca65Lexer(RegexLexer):
+    """
+    For ca65 assembler sources.
+
+    .. versionadded:: 1.6
+    """
+    name = 'ca65 assembler'
+    aliases = ['ca65']
+    filenames = ['*.s']
+
+    flags = re.IGNORECASE
+
+    tokens = {
+        'root': [
+            (r';.*', Comment.Single),
+            (r'\s+', Text),
+            (r'[a-z_.@$][\w.@$]*:', Name.Label),
+            (r'((ld|st)[axy]|(in|de)[cxy]|asl|lsr|ro[lr]|adc|sbc|cmp|cp[xy]'
+             r'|cl[cvdi]|se[cdi]|jmp|jsr|bne|beq|bpl|bmi|bvc|bvs|bcc|bcs'
+             r'|p[lh][ap]|rt[is]|brk|nop|ta[xy]|t[xy]a|txs|tsx|and|ora|eor'
+             r'|bit)\b', Keyword),
+            (r'\.\w+', Keyword.Pseudo),
+            (r'[-+~*/^&|!<>=]', Operator),
+            (r'"[^"\n]*.', String),
+            (r"'[^'\n]*.", String.Char),
+            (r'\$[0-9a-f]+|[0-9a-f]+h\b', Number.Hex),
+            (r'\d+', Number.Integer),
+            (r'%[01]+', Number.Bin),
+            (r'[#,.:()=\[\]]', Punctuation),
+            (r'[a-z_.@$][\w.@$]*', Name),
+        ]
+    }
+
+    def analyse_text(self, text):
+        # comments in GAS start with "#"
+        if re.match(r'^\s*;', text, re.MULTILINE):
+            return 0.9

packages/wakatime/packages/pygments/lexers/automation.py

@@ -5,7 +5,7 @@
 
     Lexers for automation scripting languages.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -31,8 +31,8 @@ class AutohotkeyLexer(RegexLexer):
         'root': [
             (r'^(\s*)(/\*)', bygroups(Text, Comment.Multiline), 'incomment'),
             (r'^(\s*)(\()', bygroups(Text, Generic), 'incontinuation'),
-            (r'\s+;.*?$', Comment.Singleline),
-            (r'^;.*?$', Comment.Singleline),
+            (r'\s+;.*?$', Comment.Single),
+            (r'^;.*?$', Comment.Single),
             (r'[]{}(),;[]', Punctuation),
             (r'(in|is|and|or|not)\b', Operator.Word),
             (r'\%[a-zA-Z_#@$][\w#@$]*\%', Name.Variable),
@@ -316,7 +316,8 @@ class AutoItLexer(RegexLexer):
     tokens = {
         'root': [
             (r';.*\n', Comment.Single),
-            (r'(#comments-start|#cs).*?(#comments-end|#ce)', Comment.Multiline),
+            (r'(#comments-start|#cs)(.|\n)*?(#comments-end|#ce)',
+             Comment.Multiline),
             (r'[\[\]{}(),;]', Punctuation),
             (r'(and|or|not)\b', Operator.Word),
             (r'[$|@][a-zA-Z_]\w*', Name.Variable),

packages/wakatime/packages/pygments/lexers/basic.py

@@ -5,7 +5,7 @@
 
     Lexers for BASIC like languages (other than VB.net).
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/bibtex.py

@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.bibtex
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for BibTeX bibliography data and styles
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, ExtendedRegexLexer, include, default, \
+    words
+from pygments.token import Name, Comment, String, Error, Number, Text, \
+    Keyword, Punctuation
+
+__all__ = ['BibTeXLexer', 'BSTLexer']
+
+
+class BibTeXLexer(ExtendedRegexLexer):
+    """
+    A lexer for BibTeX bibliography data format.
+
+    .. versionadded:: 2.2
+    """
+
+    name = 'BibTeX'
+    aliases = ['bib', 'bibtex']
+    filenames = ['*.bib']
+    mimetypes = ["text/x-bibtex"]
+    flags = re.IGNORECASE
+
+    ALLOWED_CHARS = r'@!$&*+\-./:;<>?\[\\\]^`|~'
+    IDENTIFIER = '[{0}][{1}]*'.format('a-z_' + ALLOWED_CHARS, r'\w' + ALLOWED_CHARS)
+
+    def open_brace_callback(self, match, ctx):
+        opening_brace = match.group()
+        ctx.opening_brace = opening_brace
+        yield match.start(), Punctuation, opening_brace
+        ctx.pos = match.end()
+
+    def close_brace_callback(self, match, ctx):
+        closing_brace = match.group()
+        if (
+            ctx.opening_brace == '{' and closing_brace != '}' or
+            ctx.opening_brace == '(' and closing_brace != ')'
+        ):
+            yield match.start(), Error, closing_brace
+        else:
+            yield match.start(), Punctuation, closing_brace
+        del ctx.opening_brace
+        ctx.pos = match.end()
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            ('@comment', Comment),
+            ('@preamble', Name.Class, ('closing-brace', 'value', 'opening-brace')),
+            ('@string', Name.Class, ('closing-brace', 'field', 'opening-brace')),
+            ('@' + IDENTIFIER, Name.Class,
+             ('closing-brace', 'command-body', 'opening-brace')),
+            ('.+', Comment),
+        ],
+        'opening-brace': [
+            include('whitespace'),
+            (r'[{(]', open_brace_callback, '#pop'),
+        ],
+        'closing-brace': [
+            include('whitespace'),
+            (r'[})]', close_brace_callback, '#pop'),
+        ],
+        'command-body': [
+            include('whitespace'),
+            (r'[^\s\,\}]+', Name.Label, ('#pop', 'fields')),
+        ],
+        'fields': [
+            include('whitespace'),
+            (',', Punctuation, 'field'),
+            default('#pop'),
+        ],
+        'field': [
+            include('whitespace'),
+            (IDENTIFIER, Name.Attribute, ('value', '=')),
+            default('#pop'),
+        ],
+        '=': [
+            include('whitespace'),
+            ('=', Punctuation, '#pop'),
+        ],
+        'value': [
+            include('whitespace'),
+            (IDENTIFIER, Name.Variable),
+            ('"', String, 'quoted-string'),
+            (r'\{', String, 'braced-string'),
+            (r'[\d]+', Number),
+            ('#', Punctuation),
+            default('#pop'),
+        ],
+        'quoted-string': [
+            (r'\{', String, 'braced-string'),
+            ('"', String, '#pop'),
+            ('[^\{\"]+', String),
+        ],
+        'braced-string': [
+            (r'\{', String, '#push'),
+            (r'\}', String, '#pop'),
+            ('[^\{\}]+', String),
+        ],
+        'whitespace': [
+            (r'\s+', Text),
+        ],
+    }
+
+
+class BSTLexer(RegexLexer):
+    """
+    A lexer for BibTeX bibliography styles.
+
+    .. versionadded:: 2.2
+    """
+
+    name = 'BST'
+    aliases = ['bst', 'bst-pybtex']
+    filenames = ['*.bst']
+    flags = re.IGNORECASE | re.MULTILINE
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            (words(['read', 'sort']), Keyword),
+            (words(['execute', 'integers', 'iterate', 'reverse', 'strings']),
+             Keyword, ('group')),
+            (words(['function', 'macro']), Keyword, ('group', 'group')),
+            (words(['entry']), Keyword, ('group', 'group', 'group')),
+        ],
+        'group': [
+            include('whitespace'),
+            (r'\{', Punctuation, ('#pop', 'group-end', 'body')),
+        ],
+        'group-end': [
+            include('whitespace'),
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'body': [
+            include('whitespace'),
+            (r"\'[^#\"\{\}\s]+", Name.Function),
+            (r'[^#\"\{\}\s]+\$', Name.Builtin),
+            (r'[^#\"\{\}\s]+', Name.Variable),
+            (r'"[^\"]*"', String),
+            (r'#-?\d+', Number),
+            (r'\{', Punctuation, ('group-end', 'body')),
+            default('#pop'),
+        ],
+        'whitespace': [
+            ('\s+', Text),
+            ('%.*?$', Comment.SingleLine),
+        ],
+    }

packages/wakatime/packages/pygments/lexers/business.py

@@ -5,7 +5,7 @@
 
     Lexers for "business-oriented" languages.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -57,9 +57,9 @@ class CobolLexer(RegexLexer):
         ],
         'core': [
             # Figurative constants
-            (r'(^|(?<=[^0-9a-z_\-]))(ALL\s+)?'
+            (r'(^|(?<=[^\w\-]))(ALL\s+)?'
              r'((ZEROES)|(HIGH-VALUE|LOW-VALUE|QUOTE|SPACE|ZERO)(S)?)'
-             r'\s*($|(?=[^0-9a-z_\-]))',
+             r'\s*($|(?=[^\w\-]))',
              Name.Constant),
 
             # Reserved words STATEMENTS and other bolds
@@ -79,8 +79,8 @@ class CobolLexer(RegexLexer):
                 'RETURN', 'REWRITE', 'SCREEN', 'SD', 'SEARCH', 'SECTION', 'SET',
                 'SORT', 'START', 'STOP', 'STRING', 'SUBTRACT', 'SUPPRESS',
                 'TERMINATE', 'THEN', 'UNLOCK', 'UNSTRING', 'USE', 'VALIDATE',
-                'WORKING-STORAGE', 'WRITE'), prefix=r'(^|(?<=[^0-9a-z_\-]))',
-                suffix=r'\s*($|(?=[^0-9a-z_\-]))'),
+                'WORKING-STORAGE', 'WRITE'), prefix=r'(^|(?<=[^\w\-]))',
+                suffix=r'\s*($|(?=[^\w\-]))'),
              Keyword.Reserved),
 
             # Reserved words
@@ -89,33 +89,33 @@ class CobolLexer(RegexLexer):
                 'ALPHABET', 'ALPHABETIC', 'ALPHABETIC-LOWER', 'ALPHABETIC-UPPER',
                 'ALPHANUMERIC', 'ALPHANUMERIC-EDITED', 'ALSO', 'ALTER', 'ALTERNATE'
                 'ANY', 'ARE', 'AREA', 'AREAS', 'ARGUMENT-NUMBER', 'ARGUMENT-VALUE', 'AS',
-                'ASCENDING', 'ASSIGN', 'AT', 'AUTO', 'AUTO-SKIP', 'AUTOMATIC', 'AUTOTERMINATE',
-                'BACKGROUND-COLOR', 'BASED', 'BEEP', 'BEFORE', 'BELL',
+                'ASCENDING', 'ASSIGN', 'AT', 'AUTO', 'AUTO-SKIP', 'AUTOMATIC',
+                'AUTOTERMINATE', 'BACKGROUND-COLOR', 'BASED', 'BEEP', 'BEFORE', 'BELL',
                 'BLANK', 'BLINK', 'BLOCK', 'BOTTOM', 'BY', 'BYTE-LENGTH', 'CHAINING',
-                'CHARACTER', 'CHARACTERS', 'CLASS', 'CODE', 'CODE-SET', 'COL', 'COLLATING',
-                'COLS', 'COLUMN', 'COLUMNS', 'COMMA', 'COMMAND-LINE', 'COMMIT', 'COMMON',
-                'CONSTANT', 'CONTAINS', 'CONTENT', 'CONTROL',
+                'CHARACTER', 'CHARACTERS', 'CLASS', 'CODE', 'CODE-SET', 'COL',
+                'COLLATING', 'COLS', 'COLUMN', 'COLUMNS', 'COMMA', 'COMMAND-LINE',
+                'COMMIT', 'COMMON', 'CONSTANT', 'CONTAINS', 'CONTENT', 'CONTROL',
                 'CONTROLS', 'CONVERTING', 'COPY', 'CORR', 'CORRESPONDING', 'COUNT', 'CRT',
-                'CURRENCY', 'CURSOR', 'CYCLE', 'DATE', 'DAY', 'DAY-OF-WEEK', 'DE', 'DEBUGGING',
-                'DECIMAL-POINT', 'DECLARATIVES', 'DEFAULT', 'DELIMITED',
+                'CURRENCY', 'CURSOR', 'CYCLE', 'DATE', 'DAY', 'DAY-OF-WEEK', 'DE',
+                'DEBUGGING', 'DECIMAL-POINT', 'DECLARATIVES', 'DEFAULT', 'DELIMITED',
                 'DELIMITER', 'DEPENDING', 'DESCENDING', 'DETAIL', 'DISK',
                 'DOWN', 'DUPLICATES', 'DYNAMIC', 'EBCDIC',
                 'ENTRY', 'ENVIRONMENT-NAME', 'ENVIRONMENT-VALUE', 'EOL', 'EOP',
                 'EOS', 'ERASE', 'ERROR', 'ESCAPE', 'EXCEPTION',
-                'EXCLUSIVE', 'EXTEND', 'EXTERNAL',
-                'FILE-ID', 'FILLER', 'FINAL', 'FIRST', 'FIXED', 'FLOAT-LONG', 'FLOAT-SHORT',
-                'FOOTING', 'FOR', 'FOREGROUND-COLOR', 'FORMAT', 'FROM', 'FULL', 'FUNCTION',
-                'FUNCTION-ID', 'GIVING', 'GLOBAL', 'GROUP',
+                'EXCLUSIVE', 'EXTEND', 'EXTERNAL', 'FILE-ID', 'FILLER', 'FINAL',
+                'FIRST', 'FIXED', 'FLOAT-LONG', 'FLOAT-SHORT',
+                'FOOTING', 'FOR', 'FOREGROUND-COLOR', 'FORMAT', 'FROM', 'FULL',
+                'FUNCTION', 'FUNCTION-ID', 'GIVING', 'GLOBAL', 'GROUP',
                 'HEADING', 'HIGHLIGHT', 'I-O', 'ID',
                 'IGNORE', 'IGNORING', 'IN', 'INDEX', 'INDEXED', 'INDICATE',
-                'INITIAL', 'INITIALIZED', 'INPUT',
-                'INTO', 'INTRINSIC', 'INVALID', 'IS', 'JUST', 'JUSTIFIED', 'KEY', 'LABEL',
+                'INITIAL', 'INITIALIZED', 'INPUT', 'INTO', 'INTRINSIC', 'INVALID',
+                'IS', 'JUST', 'JUSTIFIED', 'KEY', 'LABEL',
                 'LAST', 'LEADING', 'LEFT', 'LENGTH', 'LIMIT', 'LIMITS', 'LINAGE',
                 'LINAGE-COUNTER', 'LINE', 'LINES', 'LOCALE', 'LOCK',
-                'LOWLIGHT', 'MANUAL', 'MEMORY', 'MINUS', 'MODE',
-                'MULTIPLE', 'NATIONAL', 'NATIONAL-EDITED', 'NATIVE',
-                'NEGATIVE', 'NEXT', 'NO', 'NULL', 'NULLS', 'NUMBER', 'NUMBERS', 'NUMERIC',
-                'NUMERIC-EDITED', 'OBJECT-COMPUTER', 'OCCURS', 'OF', 'OFF', 'OMITTED', 'ON', 'ONLY',
+                'LOWLIGHT', 'MANUAL', 'MEMORY', 'MINUS', 'MODE', 'MULTIPLE',
+                'NATIONAL', 'NATIONAL-EDITED', 'NATIVE', 'NEGATIVE', 'NEXT', 'NO',
+                'NULL', 'NULLS', 'NUMBER', 'NUMBERS', 'NUMERIC', 'NUMERIC-EDITED',
+                'OBJECT-COMPUTER', 'OCCURS', 'OF', 'OFF', 'OMITTED', 'ON', 'ONLY',
                 'OPTIONAL', 'ORDER', 'ORGANIZATION', 'OTHER', 'OUTPUT', 'OVERFLOW',
                 'OVERLINE', 'PACKED-DECIMAL', 'PADDING', 'PAGE', 'PARAGRAPH',
                 'PLUS', 'POINTER', 'POSITION', 'POSITIVE', 'PRESENT', 'PREVIOUS',
@@ -137,40 +137,42 @@ class CobolLexer(RegexLexer):
                 'UNSIGNED-INT', 'UNSIGNED-LONG', 'UNSIGNED-SHORT', 'UNTIL', 'UP',
                 'UPDATE', 'UPON', 'USAGE', 'USING', 'VALUE', 'VALUES', 'VARYING',
                 'WAIT', 'WHEN', 'WITH', 'WORDS', 'YYYYDDD', 'YYYYMMDD'),
-                prefix=r'(^|(?<=[^0-9a-z_\-]))', suffix=r'\s*($|(?=[^0-9a-z_\-]))'),
+                prefix=r'(^|(?<=[^\w\-]))', suffix=r'\s*($|(?=[^\w\-]))'),
              Keyword.Pseudo),
 
             # inactive reserved words
             (words((
-                'ACTIVE-CLASS', 'ALIGNED', 'ANYCASE', 'ARITHMETIC', 'ATTRIBUTE', 'B-AND',
-                'B-NOT', 'B-OR', 'B-XOR', 'BIT', 'BOOLEAN', 'CD', 'CENTER', 'CF', 'CH', 'CHAIN', 'CLASS-ID',
-                'CLASSIFICATION', 'COMMUNICATION', 'CONDITION', 'DATA-POINTER',
-                'DESTINATION', 'DISABLE', 'EC', 'EGI', 'EMI', 'ENABLE', 'END-RECEIVE',
-                'ENTRY-CONVENTION', 'EO', 'ESI', 'EXCEPTION-OBJECT', 'EXPANDS', 'FACTORY',
-                'FLOAT-BINARY-16', 'FLOAT-BINARY-34', 'FLOAT-BINARY-7',
-                'FLOAT-DECIMAL-16', 'FLOAT-DECIMAL-34', 'FLOAT-EXTENDED', 'FORMAT',
-                'FUNCTION-POINTER', 'GET', 'GROUP-USAGE', 'IMPLEMENTS', 'INFINITY',
-                'INHERITS', 'INTERFACE', 'INTERFACE-ID', 'INVOKE', 'LC_ALL', 'LC_COLLATE',
+                'ACTIVE-CLASS', 'ALIGNED', 'ANYCASE', 'ARITHMETIC', 'ATTRIBUTE',
+                'B-AND', 'B-NOT', 'B-OR', 'B-XOR', 'BIT', 'BOOLEAN', 'CD', 'CENTER',
+                'CF', 'CH', 'CHAIN', 'CLASS-ID', 'CLASSIFICATION', 'COMMUNICATION',
+                'CONDITION', 'DATA-POINTER', 'DESTINATION', 'DISABLE', 'EC', 'EGI',
+                'EMI', 'ENABLE', 'END-RECEIVE', 'ENTRY-CONVENTION', 'EO', 'ESI',
+                'EXCEPTION-OBJECT', 'EXPANDS', 'FACTORY', 'FLOAT-BINARY-16',
+                'FLOAT-BINARY-34', 'FLOAT-BINARY-7', 'FLOAT-DECIMAL-16',
+                'FLOAT-DECIMAL-34', 'FLOAT-EXTENDED', 'FORMAT', 'FUNCTION-POINTER',
+                'GET', 'GROUP-USAGE', 'IMPLEMENTS', 'INFINITY', 'INHERITS',
+                'INTERFACE', 'INTERFACE-ID', 'INVOKE', 'LC_ALL', 'LC_COLLATE',
                 'LC_CTYPE', 'LC_MESSAGES', 'LC_MONETARY', 'LC_NUMERIC', 'LC_TIME',
-                'LINE-COUNTER', 'MESSAGE', 'METHOD', 'METHOD-ID', 'NESTED', 'NONE', 'NORMAL',
-                'OBJECT', 'OBJECT-REFERENCE', 'OPTIONS', 'OVERRIDE', 'PAGE-COUNTER', 'PF', 'PH',
-                'PROPERTY', 'PROTOTYPE', 'PURGE', 'QUEUE', 'RAISE', 'RAISING', 'RECEIVE',
-                'RELATION', 'REPLACE', 'REPRESENTS-NOT-A-NUMBER', 'RESET', 'RESUME', 'RETRY',
-                'RF', 'RH', 'SECONDS', 'SEGMENT', 'SELF', 'SEND', 'SOURCES', 'STATEMENT', 'STEP',
-                'STRONG', 'SUB-QUEUE-1', 'SUB-QUEUE-2', 'SUB-QUEUE-3', 'SUPER', 'SYMBOL',
-                'SYSTEM-DEFAULT', 'TABLE', 'TERMINAL', 'TEXT', 'TYPEDEF', 'UCS-4', 'UNIVERSAL',
-                'USER-DEFAULT', 'UTF-16', 'UTF-8', 'VAL-STATUS', 'VALID', 'VALIDATE',
-                'VALIDATE-STATUS'),
-                prefix=r'(^|(?<=[^0-9a-z_\-]))', suffix=r'\s*($|(?=[^0-9a-z_\-]))'),
+                'LINE-COUNTER', 'MESSAGE', 'METHOD', 'METHOD-ID', 'NESTED', 'NONE',
+                'NORMAL', 'OBJECT', 'OBJECT-REFERENCE', 'OPTIONS', 'OVERRIDE',
+                'PAGE-COUNTER', 'PF', 'PH', 'PROPERTY', 'PROTOTYPE', 'PURGE',
+                'QUEUE', 'RAISE', 'RAISING', 'RECEIVE', 'RELATION', 'REPLACE',
+                'REPRESENTS-NOT-A-NUMBER', 'RESET', 'RESUME', 'RETRY', 'RF', 'RH',
+                'SECONDS', 'SEGMENT', 'SELF', 'SEND', 'SOURCES', 'STATEMENT',
+                'STEP', 'STRONG', 'SUB-QUEUE-1', 'SUB-QUEUE-2', 'SUB-QUEUE-3',
+                'SUPER', 'SYMBOL', 'SYSTEM-DEFAULT', 'TABLE', 'TERMINAL', 'TEXT',
+                'TYPEDEF', 'UCS-4', 'UNIVERSAL', 'USER-DEFAULT', 'UTF-16', 'UTF-8',
+                'VAL-STATUS', 'VALID', 'VALIDATE', 'VALIDATE-STATUS'),
+                   prefix=r'(^|(?<=[^\w\-]))', suffix=r'\s*($|(?=[^\w\-]))'),
              Error),
 
             # Data Types
-            (r'(^|(?<=[^0-9a-z_\-]))'
+            (r'(^|(?<=[^\w\-]))'
              r'(PIC\s+.+?(?=(\s|\.\s))|PICTURE\s+.+?(?=(\s|\.\s))|'
              r'(COMPUTATIONAL)(-[1-5X])?|(COMP)(-[1-5X])?|'
              r'BINARY-C-LONG|'
              r'BINARY-CHAR|BINARY-DOUBLE|BINARY-LONG|BINARY-SHORT|'
-             r'BINARY)\s*($|(?=[^0-9a-z_\-]))', Keyword.Type),
+             r'BINARY)\s*($|(?=[^\w\-]))', Keyword.Type),
 
             # Operators
             (r'(\*\*|\*|\+|-|/|<=|>=|<|>|==|/=|=)', Operator),
@@ -180,7 +182,7 @@ class CobolLexer(RegexLexer):
             (r'([(),;:&%.])', Punctuation),
 
             # Intrinsics
-            (r'(^|(?<=[^0-9a-z_\-]))(ABS|ACOS|ANNUITY|ASIN|ATAN|BYTE-LENGTH|'
+            (r'(^|(?<=[^\w\-]))(ABS|ACOS|ANNUITY|ASIN|ATAN|BYTE-LENGTH|'
              r'CHAR|COMBINED-DATETIME|CONCATENATE|COS|CURRENT-DATE|'
              r'DATE-OF-INTEGER|DATE-TO-YYYYMMDD|DAY-OF-INTEGER|DAY-TO-YYYYDDD|'
              r'EXCEPTION-(?:FILE|LOCATION|STATEMENT|STATUS)|EXP10|EXP|E|'
@@ -192,13 +194,13 @@ class CobolLexer(RegexLexer):
              r'STANDARD-DEVIATION|STORED-CHAR-LENGTH|SUBSTITUTE(?:-CASE)?|'
              r'SUM|TAN|TEST-DATE-YYYYMMDD|TEST-DAY-YYYYDDD|TRIM|'
              r'UPPER-CASE|VARIANCE|WHEN-COMPILED|YEAR-TO-YYYY)\s*'
-             r'($|(?=[^0-9a-z_\-]))', Name.Function),
+             r'($|(?=[^\w\-]))', Name.Function),
 
             # Booleans
-            (r'(^|(?<=[^0-9a-z_\-]))(true|false)\s*($|(?=[^0-9a-z_\-]))', Name.Builtin),
+            (r'(^|(?<=[^\w\-]))(true|false)\s*($|(?=[^\w\-]))', Name.Builtin),
             # Comparing Operators
-            (r'(^|(?<=[^0-9a-z_\-]))(equal|equals|ne|lt|le|gt|ge|'
-             r'greater|less|than|not|and|or)\s*($|(?=[^0-9a-z_\-]))', Operator.Word),
+            (r'(^|(?<=[^\w\-]))(equal|equals|ne|lt|le|gt|ge|'
+             r'greater|less|than|not|and|or)\s*($|(?=[^\w\-]))', Operator.Word),
         ],
 
         # \"[^\"\n]*\"|\'[^\'\n]*\'
@@ -244,7 +246,7 @@ class ABAPLexer(RegexLexer):
     """
     name = 'ABAP'
     aliases = ['abap']
-    filenames = ['*.abap']
+    filenames = ['*.abap', '*.ABAP']
     mimetypes = ['text/x-abap']
 
     flags = re.IGNORECASE | re.MULTILINE
@@ -254,6 +256,7 @@ class ABAPLexer(RegexLexer):
             (r'\s+', Text),
             (r'^\*.*$', Comment.Single),
             (r'\".*?\n', Comment.Single),
+            (r'##\w+', Comment.Special),
         ],
         'variable-names': [
             (r'<\S+>', Name.Variable),
@@ -262,8 +265,8 @@ class ABAPLexer(RegexLexer):
         'root': [
             include('common'),
             # function calls
-            (r'(CALL\s+(?:BADI|CUSTOMER-FUNCTION|FUNCTION))(\s+)(\'?\S+\'?)',
-             bygroups(Keyword, Text, Name.Function)),
+            (r'CALL\s+(?:BADI|CUSTOMER-FUNCTION|FUNCTION)',
+             Keyword),
             (r'(CALL\s+(?:DIALOG|SCREEN|SUBSCREEN|SELECTION-SCREEN|'
              r'TRANSACTION|TRANSFORMATION))\b',
              Keyword),
@@ -283,6 +286,12 @@ class ABAPLexer(RegexLexer):
             # call methodnames returning style
             (r'(?<=(=|-)>)([\w\-~]+)(?=\()', Name.Function),
 
+            # text elements
+            (r'(TEXT)(-)(\d{3})',
+             bygroups(Keyword, Punctuation, Number.Integer)),
+            (r'(TEXT)(-)(\w{3})',
+             bygroups(Keyword, Punctuation, Name.Variable)),
+
             # keywords with dashes in them.
             # these need to be first, because for instance the -ID part
             # of MESSAGE-ID wouldn't get highlighted if MESSAGE was
@@ -299,13 +308,13 @@ class ABAPLexer(RegexLexer):
              r'OUTPUT-LENGTH|PRINT-CONTROL|'
              r'SELECT-OPTIONS|START-OF-SELECTION|SUBTRACT-CORRESPONDING|'
              r'SYNTAX-CHECK|SYSTEM-EXCEPTIONS|'
-             r'TYPE-POOL|TYPE-POOLS'
+             r'TYPE-POOL|TYPE-POOLS|NO-DISPLAY'
              r')\b', Keyword),
 
             # keyword kombinations
-            (r'CREATE\s+(PUBLIC|PRIVATE|DATA|OBJECT)|'
-             r'((PUBLIC|PRIVATE|PROTECTED)\s+SECTION|'
-             r'(TYPE|LIKE)(\s+(LINE\s+OF|REF\s+TO|'
+            (r'(?<![-\>])(CREATE\s+(PUBLIC|PRIVATE|DATA|OBJECT)|'
+             r'(PUBLIC|PRIVATE|PROTECTED)\s+SECTION|'
+             r'(TYPE|LIKE)\s+((LINE\s+OF|REF\s+TO|'
              r'(SORTED|STANDARD|HASHED)\s+TABLE\s+OF))?|'
              r'FROM\s+(DATABASE|MEMORY)|CALL\s+METHOD|'
              r'(GROUP|ORDER) BY|HAVING|SEPARATED BY|'
@@ -341,10 +350,16 @@ class ABAPLexer(RegexLexer):
              r'(BEGIN|END)\s+OF|'
              r'DELETE(\s+ADJACENT\s+DUPLICATES\sFROM)?|'
              r'COMPARING(\s+ALL\s+FIELDS)?|'
-             r'INSERT(\s+INITIAL\s+LINE\s+INTO|\s+LINES\s+OF)?|'
+             r'(INSERT|APPEND)(\s+INITIAL\s+LINE\s+(IN)?TO|\s+LINES\s+OF)?|'
              r'IN\s+((BYTE|CHARACTER)\s+MODE|PROGRAM)|'
              r'END-OF-(DEFINITION|PAGE|SELECTION)|'
              r'WITH\s+FRAME(\s+TITLE)|'
+             r'(REPLACE|FIND)\s+((FIRST|ALL)\s+OCCURRENCES?\s+OF\s+)?(SUBSTRING|REGEX)?|'
+             r'MATCH\s+(LENGTH|COUNT|LINE|OFFSET)|'
+             r'(RESPECTING|IGNORING)\s+CASE|'
+             r'IN\s+UPDATE\s+TASK|'
+             r'(SOURCE|RESULT)\s+(XML)?|'
+             r'REFERENCE\s+INTO|'
 
              # simple kombinations
              r'AND\s+(MARK|RETURN)|CLIENT\s+SPECIFIED|CORRESPONDING\s+FIELDS\s+OF|'
@@ -353,39 +368,41 @@ class ABAPLexer(RegexLexer):
              r'MODIFY\s+SCREEN|NESTING\s+LEVEL|NO\s+INTERVALS|OF\s+STRUCTURE|'
              r'RADIOBUTTON\s+GROUP|RANGE\s+OF|REF\s+TO|SUPPRESS DIALOG|'
              r'TABLE\s+OF|UPPER\s+CASE|TRANSPORTING\s+NO\s+FIELDS|'
-             r'VALUE\s+CHECK|VISIBLE\s+LENGTH|HEADER\s+LINE)\b', Keyword),
+             r'VALUE\s+CHECK|VISIBLE\s+LENGTH|HEADER\s+LINE|COMMON\s+PART)\b', Keyword),
 
             # single word keywords.
-            (r'(^|(?<=(\s|\.)))(ABBREVIATED|ADD|ALIASES|APPEND|ASSERT|'
-             r'ASSIGN(ING)?|AT(\s+FIRST)?|'
+            (r'(^|(?<=(\s|\.)))(ABBREVIATED|ABSTRACT|ADD|ALIASES|ALIGN|ALPHA|'
+             r'ASSERT|AS|ASSIGN(ING)?|AT(\s+FIRST)?|'
              r'BACK|BLOCK|BREAK-POINT|'
              r'CASE|CATCH|CHANGING|CHECK|CLASS|CLEAR|COLLECT|COLOR|COMMIT|'
              r'CREATE|COMMUNICATION|COMPONENTS?|COMPUTE|CONCATENATE|CONDENSE|'
-             r'CONSTANTS|CONTEXTS|CONTINUE|CONTROLS|'
-             r'DATA|DECIMALS|DEFAULT|DEFINE|DEFINITION|DEFERRED|DEMAND|'
-             r'DETAIL|DIRECTORY|DIVIDE|DO|'
-             r'ELSE(IF)?|ENDAT|ENDCASE|ENDCLASS|ENDDO|ENDFORM|ENDFUNCTION|'
-             r'ENDIF|ENDLOOP|ENDMETHOD|ENDMODULE|ENDSELECT|ENDTRY|'
-             r'ENHANCEMENT|EVENTS|EXCEPTIONS|EXIT|EXPORT|EXPORTING|EXTRACT|'
-             r'FETCH|FIELDS?|FIND|FOR|FORM|FORMAT|FREE|FROM|'
+             r'CONSTANTS|CONTEXTS|CONTINUE|CONTROLS|COUNTRY|CURRENCY|'
+             r'DATA|DATE|DECIMALS|DEFAULT|DEFINE|DEFINITION|DEFERRED|DEMAND|'
+             r'DETAIL|DIRECTORY|DIVIDE|DO|DUMMY|'
+             r'ELSE(IF)?|ENDAT|ENDCASE|ENDCATCH|ENDCLASS|ENDDO|ENDFORM|ENDFUNCTION|'
+             r'ENDIF|ENDINTERFACE|ENDLOOP|ENDMETHOD|ENDMODULE|ENDSELECT|ENDTRY|ENDWHILE|'
+             r'ENHANCEMENT|EVENTS|EXACT|EXCEPTIONS?|EXIT|EXPONENT|EXPORT|EXPORTING|EXTRACT|'
+             r'FETCH|FIELDS?|FOR|FORM|FORMAT|FREE|FROM|FUNCTION|'
              r'HIDE|'
              r'ID|IF|IMPORT|IMPLEMENTATION|IMPORTING|IN|INCLUDE|INCLUDING|'
              r'INDEX|INFOTYPES|INITIALIZATION|INTERFACE|INTERFACES|INTO|'
-             r'LENGTH|LINES|LOAD|LOCAL|'
+             r'LANGUAGE|LEAVE|LENGTH|LINES|LOAD|LOCAL|'
              r'JOIN|'
              r'KEY|'
-             r'MAXIMUM|MESSAGE|METHOD[S]?|MINIMUM|MODULE|MODIFY|MOVE|MULTIPLY|'
-             r'NODES|'
-             r'OBLIGATORY|OF|OFF|ON|OVERLAY|'
-             r'PACK|PARAMETERS|PERCENTAGE|POSITION|PROGRAM|PROVIDE|PUBLIC|PUT|'
-             r'RAISE|RAISING|RANGES|READ|RECEIVE|REFRESH|REJECT|REPORT|RESERVE|'
-             r'RESUME|RETRY|RETURN|RETURNING|RIGHT|ROLLBACK|'
-             r'SCROLL|SEARCH|SELECT|SHIFT|SINGLE|SKIP|SORT|SPLIT|STATICS|STOP|'
-             r'SUBMIT|SUBTRACT|SUM|SUMMARY|SUMMING|SUPPLY|'
-             r'TABLE|TABLES|TIMES|TITLE|TO|TOP-OF-PAGE|TRANSFER|TRANSLATE|TRY|TYPES|'
+             r'NEXT|'
+             r'MAXIMUM|MESSAGE|METHOD[S]?|MINIMUM|MODULE|MODIFIER|MODIFY|MOVE|MULTIPLY|'
+             r'NODES|NUMBER|'
+             r'OBLIGATORY|OBJECT|OF|OFF|ON|OTHERS|OVERLAY|'
+             r'PACK|PAD|PARAMETERS|PERCENTAGE|POSITION|PROGRAM|PROVIDE|PUBLIC|PUT|PF\d\d|'
+             r'RAISE|RAISING|RANGES?|READ|RECEIVE|REDEFINITION|REFRESH|REJECT|REPORT|RESERVE|'
+             r'RESUME|RETRY|RETURN|RETURNING|RIGHT|ROLLBACK|REPLACE|'
+             r'SCROLL|SEARCH|SELECT|SHIFT|SIGN|SINGLE|SIZE|SKIP|SORT|SPLIT|STATICS|STOP|'
+             r'STYLE|SUBMATCHES|SUBMIT|SUBTRACT|SUM(?!\()|SUMMARY|SUMMING|SUPPLY|'
+             r'TABLE|TABLES|TIMESTAMP|TIMES?|TIMEZONE|TITLE|\??TO|'
+             r'TOP-OF-PAGE|TRANSFER|TRANSLATE|TRY|TYPES|'
              r'ULINE|UNDER|UNPACK|UPDATE|USING|'
-             r'VALUE|VALUES|VIA|'
-             r'WAIT|WHEN|WHERE|WHILE|WITH|WINDOW|WRITE)\b', Keyword),
+             r'VALUE|VALUES|VIA|VARYING|VARY|'
+             r'WAIT|WHEN|WHERE|WIDTH|WHILE|WITH|WINDOW|WRITE|XSD|ZERO)\b', Keyword),
 
             # builtins
             (r'(abs|acos|asin|atan|'
@@ -411,18 +428,21 @@ class ABAPLexer(RegexLexer):
 
             # operators which look like variable names before
             # parsing variable names.
-            (r'(?<=(\s|.))(AND|EQ|NE|GT|LT|GE|LE|CO|CN|CA|NA|CS|NOT|NS|CP|NP|'
+            (r'(?<=(\s|.))(AND|OR|EQ|NE|GT|LT|GE|LE|CO|CN|CA|NA|CS|NOT|NS|CP|NP|'
              r'BYTE-CO|BYTE-CN|BYTE-CA|BYTE-NA|BYTE-CS|BYTE-NS|'
-             r'IS\s+(NOT\s+)?(INITIAL|ASSIGNED|REQUESTED|BOUND))\b', Operator),
+             r'IS\s+(NOT\s+)?(INITIAL|ASSIGNED|REQUESTED|BOUND))\b', Operator.Word),
 
             include('variable-names'),
 
-            # standard oparators after variable names,
+            # standard operators after variable names,
             # because < and > are part of field symbols.
-            (r'[?*<>=\-+]', Operator),
+            (r'[?*<>=\-+&]', Operator),
             (r"'(''|[^'])*'", String.Single),
             (r"`([^`])*`", String.Single),
-            (r'[/;:()\[\],.]', Punctuation)
+            (r"([|}])([^{}|]*?)([|{])",
+             bygroups(Punctuation, String.Single, Punctuation)),
+            (r'[/;:()\[\],.]', Punctuation),
+            (r'(!)(\w+)', bygroups(Operator, Name)),
         ],
     }
 
@@ -439,15 +459,15 @@ class OpenEdgeLexer(RegexLexer):
     filenames = ['*.p', '*.cls']
     mimetypes = ['text/x-openedge', 'application/x-openedge']
 
-    types = (r'(?i)(^|(?<=[^0-9a-z_\-]))(CHARACTER|CHAR|CHARA|CHARAC|CHARACT|CHARACTE|'
+    types = (r'(?i)(^|(?<=[^\w\-]))(CHARACTER|CHAR|CHARA|CHARAC|CHARACT|CHARACTE|'
              r'COM-HANDLE|DATE|DATETIME|DATETIME-TZ|'
              r'DECIMAL|DEC|DECI|DECIM|DECIMA|HANDLE|'
              r'INT64|INTEGER|INT|INTE|INTEG|INTEGE|'
-             r'LOGICAL|LONGCHAR|MEMPTR|RAW|RECID|ROWID)\s*($|(?=[^0-9a-z_\-]))')
+             r'LOGICAL|LONGCHAR|MEMPTR|RAW|RECID|ROWID)\s*($|(?=[^\w\-]))')
 
     keywords = words(OPENEDGEKEYWORDS,
-                     prefix=r'(?i)(^|(?<=[^0-9a-z_\-]))',
-                     suffix=r'\s*($|(?=[^0-9a-z_\-]))')
+                     prefix=r'(?i)(^|(?<=[^\w\-]))',
+                     suffix=r'\s*($|(?=[^\w\-]))')
 
     tokens = {
         'root': [

packages/wakatime/packages/pygments/lexers/c_cpp.py

@@ -5,7 +5,7 @@
 
     Lexers for C/C++ languages.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -28,8 +28,10 @@ class CFamilyLexer(RegexLexer):
 
     #: optional Comment or Whitespace
     _ws = r'(?:\s|//.*?\n|/[*].*?[*]/)+'
+
+    # The trailing ?, rather than *, avoids a geometric performance drop here.
     #: only one /* */ style comment
-    _ws1 = r'\s*(?:/[*].*?[*]/\s*)*'
+    _ws1 = r'\s*(?:/[*].*?[*]/\s*)?'
 
     tokens = {
         'whitespace': [
@@ -44,12 +46,15 @@ class CFamilyLexer(RegexLexer):
             (r'\n', Text),
             (r'\s+', Text),
             (r'\\\n', Text),  # line continuation
-            (r'//(\n|(.|\n)*?[^\\]\n)', Comment.Single),
-            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+            (r'//(\n|[\w\W]*?[^\\]\n)', Comment.Single),
+            (r'/(\\\n)?[*][\w\W]*?[*](\\\n)?/', Comment.Multiline),
+            # Open until EOF, so no ending delimeter
+            (r'/(\\\n)?[*][\w\W]*', Comment.Multiline),
         ],
         'statements': [
-            (r'L?"', String, 'string'),
-            (r"L?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char),
+            (r'(L?)(")', bygroups(String.Affix, String), 'string'),
+            (r"(L?)(')(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])(')",
+             bygroups(String.Affix, String.Char, String.Char, String.Char)),
             (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*', Number.Float),
             (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
             (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex),
@@ -58,13 +63,13 @@ class CFamilyLexer(RegexLexer):
             (r'\*/', Error),
             (r'[~!%^&*+=|?:<>/-]', Operator),
             (r'[()\[\],.]', Punctuation),
-            (words(('auto', 'break', 'case', 'const', 'continue', 'default', 'do',
-                    'else', 'enum', 'extern', 'for', 'goto', 'if', 'register',
-                    'restricted', 'return', 'sizeof', 'static', 'struct',
-                    'switch', 'typedef', 'union', 'volatile', 'while'),
+            (words(('asm', 'auto', 'break', 'case', 'const', 'continue',
+                    'default', 'do', 'else', 'enum', 'extern', 'for', 'goto',
+                    'if', 'register', 'restricted', 'return', 'sizeof',
+                    'static', 'struct', 'switch', 'typedef', 'union',
+                    'volatile', 'while'),
                    suffix=r'\b'), Keyword),
-            (r'(bool|int|long|float|short|double|char|unsigned|signed|void|'
-             r'[a-z_][a-z0-9_]*_t)\b',
+            (r'(bool|int|long|float|short|double|char|unsigned|signed|void)\b',
              Keyword.Type),
             (words(('inline', '_inline', '__inline', 'naked', 'restrict',
                     'thread', 'typename'), suffix=r'\b'), Keyword.Reserved),
@@ -87,7 +92,7 @@ class CFamilyLexer(RegexLexer):
             (r'((?:[\w*\s])+?(?:\s|[*]))'  # return arguments
              r'([a-zA-Z_]\w*)'             # method name
              r'(\s*\([^;]*?\))'            # signature
-             r'(' + _ws + r')?(\{)',
+             r'([^;{]*)(\{)',
              bygroups(using(this), Name.Function, using(this), using(this),
                       Punctuation),
              'function'),
@@ -95,7 +100,7 @@ class CFamilyLexer(RegexLexer):
             (r'((?:[\w*\s])+?(?:\s|[*]))'  # return arguments
              r'([a-zA-Z_]\w*)'             # method name
              r'(\s*\([^;]*?\))'            # signature
-             r'(' + _ws + r')?(;)',
+             r'([^;]*)(;)',
              bygroups(using(this), Name.Function, using(this), using(this),
                       Punctuation)),
             default('statement'),
@@ -122,6 +127,8 @@ class CFamilyLexer(RegexLexer):
             (r'\\', String),  # stray backslash
         ],
         'macro': [
+            (r'(include)(' + _ws1 + r')([^\n]+)',
+             bygroups(Comment.Preproc, Text, Comment.PreprocFile)),
             (r'[^/\n]+', Comment.Preproc),
             (r'/[*](.|\n)*?[*]/', Comment.Multiline),
             (r'//.*?\n', Comment.Single, '#pop'),
@@ -137,22 +144,26 @@ class CFamilyLexer(RegexLexer):
         ]
     }
 
-    stdlib_types = ['size_t', 'ssize_t', 'off_t', 'wchar_t', 'ptrdiff_t',
-                    'sig_atomic_t', 'fpos_t', 'clock_t', 'time_t', 'va_list',
-                    'jmp_buf', 'FILE', 'DIR', 'div_t', 'ldiv_t', 'mbstate_t',
-                    'wctrans_t', 'wint_t', 'wctype_t']
-    c99_types = ['_Bool', '_Complex', 'int8_t', 'int16_t', 'int32_t', 'int64_t',
-                 'uint8_t', 'uint16_t', 'uint32_t', 'uint64_t', 'int_least8_t',
-                 'int_least16_t', 'int_least32_t', 'int_least64_t',
-                 'uint_least8_t', 'uint_least16_t', 'uint_least32_t',
-                 'uint_least64_t', 'int_fast8_t', 'int_fast16_t', 'int_fast32_t',
-                 'int_fast64_t', 'uint_fast8_t', 'uint_fast16_t', 'uint_fast32_t',
-                 'uint_fast64_t', 'intptr_t', 'uintptr_t', 'intmax_t',
-                 'uintmax_t']
+    stdlib_types = set((
+        'size_t', 'ssize_t', 'off_t', 'wchar_t', 'ptrdiff_t', 'sig_atomic_t', 'fpos_t',
+        'clock_t', 'time_t', 'va_list', 'jmp_buf', 'FILE', 'DIR', 'div_t', 'ldiv_t',
+        'mbstate_t', 'wctrans_t', 'wint_t', 'wctype_t'))
+    c99_types = set((
+        '_Bool', '_Complex', 'int8_t', 'int16_t', 'int32_t', 'int64_t', 'uint8_t',
+        'uint16_t', 'uint32_t', 'uint64_t', 'int_least8_t', 'int_least16_t',
+        'int_least32_t', 'int_least64_t', 'uint_least8_t', 'uint_least16_t',
+        'uint_least32_t', 'uint_least64_t', 'int_fast8_t', 'int_fast16_t', 'int_fast32_t',
+        'int_fast64_t', 'uint_fast8_t', 'uint_fast16_t', 'uint_fast32_t', 'uint_fast64_t',
+        'intptr_t', 'uintptr_t', 'intmax_t', 'uintmax_t'))
+    linux_types = set((
+        'clockid_t', 'cpu_set_t', 'cpumask_t', 'dev_t', 'gid_t', 'id_t', 'ino_t', 'key_t',
+        'mode_t', 'nfds_t', 'pid_t', 'rlim_t', 'sig_t', 'sighandler_t', 'siginfo_t',
+        'sigset_t', 'sigval_t', 'socklen_t', 'timer_t', 'uid_t'))
 
     def __init__(self, **options):
         self.stdlibhighlighting = get_bool_opt(options, 'stdlibhighlighting', True)
         self.c99highlighting = get_bool_opt(options, 'c99highlighting', True)
+        self.platformhighlighting = get_bool_opt(options, 'platformhighlighting', True)
         RegexLexer.__init__(self, **options)
 
     def get_tokens_unprocessed(self, text):
@@ -163,6 +174,8 @@ class CFamilyLexer(RegexLexer):
                     token = Keyword.Type
                 elif self.c99highlighting and value in self.c99_types:
                     token = Keyword.Type
+                elif self.platformhighlighting and value in self.linux_types:
+                    token = Keyword.Type
             yield index, token, value
 
 
@@ -179,7 +192,7 @@ class CLexer(CFamilyLexer):
     def analyse_text(text):
         if re.search('^\s*#include [<"]', text, re.MULTILINE):
             return 0.1
-        if re.search('^\s*#ifdef ', text, re.MULTILINE):
+        if re.search('^\s*#ifn?def ', text, re.MULTILINE):
             return 0.1
 
 
@@ -198,16 +211,22 @@ class CppLexer(CFamilyLexer):
     tokens = {
         'statements': [
             (words((
-                'asm', 'catch', 'const_cast', 'delete', 'dynamic_cast', 'explicit',
+                'catch', 'const_cast', 'delete', 'dynamic_cast', 'explicit',
                 'export', 'friend', 'mutable', 'namespace', 'new', 'operator',
                 'private', 'protected', 'public', 'reinterpret_cast',
                 'restrict', 'static_cast', 'template', 'this', 'throw', 'throws',
-                'typeid', 'typename', 'using', 'virtual',
+                'try', 'typeid', 'typename', 'using', 'virtual',
                 'constexpr', 'nullptr', 'decltype', 'thread_local',
                 'alignas', 'alignof', 'static_assert', 'noexcept', 'override',
                 'final'), suffix=r'\b'), Keyword),
             (r'char(16_t|32_t)\b', Keyword.Type),
             (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'),
+            # C++11 raw strings
+            (r'(R)(")([^\\()\s]{,16})(\()((?:.|\n)*?)(\)\3)(")',
+             bygroups(String.Affix, String, String.Delimiter, String.Delimiter,
+                      String, String.Delimiter, String)),
+            # C++11 UTF-8/16/32 strings
+            (r'(u8|u|U)(")', bygroups(String.Affix, String), 'string'),
             inherit,
         ],
         'root': [
@@ -227,7 +246,7 @@ class CppLexer(CFamilyLexer):
     }
 
     def analyse_text(text):
-        if re.search('#include <[a-z]+>', text):
+        if re.search('#include <[a-z_]+>', text):
             return 0.2
         if re.search('using namespace ', text):
             return 0.4

packages/wakatime/packages/pygments/lexers/c_like.py

@@ -5,7 +5,7 @@
 
     Lexers for other C-like languages.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -20,7 +20,7 @@ from pygments.lexers.c_cpp import CLexer, CppLexer
 from pygments.lexers import _mql_builtins
 
 __all__ = ['PikeLexer', 'NesCLexer', 'ClayLexer', 'ECLexer', 'ValaLexer',
-           'CudaLexer', 'SwigLexer', 'MqlLexer']
+           'CudaLexer', 'SwigLexer', 'MqlLexer', 'ArduinoLexer']
 
 
 class PikeLexer(CppLexer):
@@ -105,7 +105,7 @@ class ClayLexer(RegexLexer):
     tokens = {
         'root': [
             (r'\s', Text),
-            (r'//.*?$', Comment.Singleline),
+            (r'//.*?$', Comment.Single),
             (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
             (r'\b(public|private|import|as|record|variant|instance'
              r'|define|overload|default|external|alias'
@@ -411,3 +411,131 @@ class MqlLexer(CppLexer):
             inherit,
         ],
     }
+
+class ArduinoLexer(CppLexer):
+    """
+    For `Arduino(tm) <https://arduino.cc/>`_ source.
+
+    This is an extension of the CppLexer, as the Arduino® Language is a superset
+    of C++
+
+    .. versionadded:: 2.1
+    """
+
+    name = 'Arduino'
+    aliases = ['arduino']
+    filenames = ['*.ino']
+    mimetypes = ['text/x-arduino']
+
+    # Language sketch main structure functions
+    structure = set(('setup', 'loop'))
+
+    # Language operators
+    operators = set(('not', 'or', 'and', 'xor'))
+
+    # Language 'variables'
+    variables = set((
+        'DIGITAL_MESSAGE', 'FIRMATA_STRING', 'ANALOG_MESSAGE', 'REPORT_DIGITAL',
+        'REPORT_ANALOG', 'INPUT_PULLUP', 'SET_PIN_MODE', 'INTERNAL2V56', 'SYSTEM_RESET',
+        'LED_BUILTIN', 'INTERNAL1V1', 'SYSEX_START', 'INTERNAL', 'EXTERNAL', 'HIGH',
+        'LOW', 'INPUT', 'OUTPUT', 'INPUT_PULLUP', 'LED_BUILTIN', 'true', 'false',
+        'void', 'boolean', 'char', 'unsigned char', 'byte', 'int', 'unsigned int',
+        'word', 'long', 'unsigned long', 'short', 'float', 'double', 'string', 'String',
+        'array', 'static', 'volatile', 'const', 'boolean', 'byte', 'word', 'string',
+        'String', 'array', 'int', 'float', 'private', 'char', 'virtual', 'operator',
+        'sizeof', 'uint8_t', 'uint16_t', 'uint32_t', 'uint64_t', 'int8_t', 'int16_t',
+        'int32_t', 'int64_t', 'dynamic_cast', 'typedef', 'const_cast', 'const',
+        'struct', 'static_cast', 'union', 'unsigned', 'long', 'volatile', 'static',
+        'protected', 'bool', 'public', 'friend', 'auto', 'void', 'enum', 'extern',
+        'class', 'short', 'reinterpret_cast', 'double', 'register', 'explicit',
+        'signed', 'inline', 'delete', '_Bool', 'complex', '_Complex', '_Imaginary',
+        'atomic_bool', 'atomic_char', 'atomic_schar', 'atomic_uchar', 'atomic_short',
+        'atomic_ushort', 'atomic_int', 'atomic_uint', 'atomic_long', 'atomic_ulong',
+        'atomic_llong', 'atomic_ullong', 'PROGMEM'))
+
+    # Language shipped functions and class ( )
+    functions = set((
+        'KeyboardController', 'MouseController', 'SoftwareSerial', 'EthernetServer',
+        'EthernetClient', 'LiquidCrystal', 'RobotControl', 'GSMVoiceCall',
+        'EthernetUDP', 'EsploraTFT', 'HttpClient', 'RobotMotor', 'WiFiClient',
+        'GSMScanner', 'FileSystem', 'Scheduler', 'GSMServer', 'YunClient', 'YunServer',
+        'IPAddress', 'GSMClient', 'GSMModem', 'Keyboard', 'Ethernet', 'Console',
+        'GSMBand', 'Esplora', 'Stepper', 'Process', 'WiFiUDP', 'GSM_SMS', 'Mailbox',
+        'USBHost', 'Firmata', 'PImage', 'Client', 'Server', 'GSMPIN', 'FileIO',
+        'Bridge', 'Serial', 'EEPROM', 'Stream', 'Mouse', 'Audio', 'Servo', 'File',
+        'Task', 'GPRS', 'WiFi', 'Wire', 'TFT', 'GSM', 'SPI', 'SD',
+        'runShellCommandAsynchronously', 'analogWriteResolution',
+        'retrieveCallingNumber', 'printFirmwareVersion', 'analogReadResolution',
+        'sendDigitalPortPair', 'noListenOnLocalhost', 'readJoystickButton',
+        'setFirmwareVersion', 'readJoystickSwitch', 'scrollDisplayRight',
+        'getVoiceCallStatus', 'scrollDisplayLeft', 'writeMicroseconds',
+        'delayMicroseconds', 'beginTransmission', 'getSignalStrength',
+        'runAsynchronously', 'getAsynchronously', 'listenOnLocalhost',
+        'getCurrentCarrier', 'readAccelerometer', 'messageAvailable',
+        'sendDigitalPorts', 'lineFollowConfig', 'countryNameWrite', 'runShellCommand',
+        'readStringUntil', 'rewindDirectory', 'readTemperature', 'setClockDivider',
+        'readLightSensor', 'endTransmission', 'analogReference', 'detachInterrupt',
+        'countryNameRead', 'attachInterrupt', 'encryptionType', 'readBytesUntil',
+        'robotNameWrite', 'readMicrophone', 'robotNameRead', 'cityNameWrite',
+        'userNameWrite', 'readJoystickY', 'readJoystickX', 'mouseReleased',
+        'openNextFile', 'scanNetworks', 'noInterrupts', 'digitalWrite', 'beginSpeaker',
+        'mousePressed', 'isActionDone', 'mouseDragged', 'displayLogos', 'noAutoscroll',
+        'addParameter', 'remoteNumber', 'getModifiers', 'keyboardRead', 'userNameRead',
+        'waitContinue', 'processInput', 'parseCommand', 'printVersion', 'readNetworks',
+        'writeMessage', 'blinkVersion', 'cityNameRead', 'readMessage', 'setDataMode',
+        'parsePacket', 'isListening', 'setBitOrder', 'beginPacket', 'isDirectory',
+        'motorsWrite', 'drawCompass', 'digitalRead', 'clearScreen', 'serialEvent',
+        'rightToLeft', 'setTextSize', 'leftToRight', 'requestFrom', 'keyReleased',
+        'compassRead', 'analogWrite', 'interrupts', 'WiFiServer', 'disconnect',
+        'playMelody', 'parseFloat', 'autoscroll', 'getPINUsed', 'setPINUsed',
+        'setTimeout', 'sendAnalog', 'readSlider', 'analogRead', 'beginWrite',
+        'createChar', 'motorsStop', 'keyPressed', 'tempoWrite', 'readButton',
+        'subnetMask', 'debugPrint', 'macAddress', 'writeGreen', 'randomSeed',
+        'attachGPRS', 'readString', 'sendString', 'remotePort', 'releaseAll',
+        'mouseMoved', 'background', 'getXChange', 'getYChange', 'answerCall',
+        'getResult', 'voiceCall', 'endPacket', 'constrain', 'getSocket', 'writeJSON',
+        'getButton', 'available', 'connected', 'findUntil', 'readBytes', 'exitValue',
+        'readGreen', 'writeBlue', 'startLoop', 'IPAddress', 'isPressed', 'sendSysex',
+        'pauseMode', 'gatewayIP', 'setCursor', 'getOemKey', 'tuneWrite', 'noDisplay',
+        'loadImage', 'switchPIN', 'onRequest', 'onReceive', 'changePIN', 'playFile',
+        'noBuffer', 'parseInt', 'overflow', 'checkPIN', 'knobRead', 'beginTFT',
+        'bitClear', 'updateIR', 'bitWrite', 'position', 'writeRGB', 'highByte',
+        'writeRed', 'setSpeed', 'readBlue', 'noStroke', 'remoteIP', 'transfer',
+        'shutdown', 'hangCall', 'beginSMS', 'endWrite', 'attached', 'maintain',
+        'noCursor', 'checkReg', 'checkPUK', 'shiftOut', 'isValid', 'shiftIn', 'pulseIn',
+        'connect', 'println', 'localIP', 'pinMode', 'getIMEI', 'display', 'noBlink',
+        'process', 'getBand', 'running', 'beginSD', 'drawBMP', 'lowByte', 'setBand',
+        'release', 'bitRead', 'prepare', 'pointTo', 'readRed', 'setMode', 'noFill',
+        'remove', 'listen', 'stroke', 'detach', 'attach', 'noTone', 'exists', 'buffer',
+        'height', 'bitSet', 'circle', 'config', 'cursor', 'random', 'IRread', 'setDNS',
+        'endSMS', 'getKey', 'micros', 'millis', 'begin', 'print', 'write', 'ready',
+        'flush', 'width', 'isPIN', 'blink', 'clear', 'press', 'mkdir', 'rmdir', 'close',
+        'point', 'yield', 'image', 'BSSID', 'click', 'delay', 'read', 'text', 'move',
+        'peek', 'beep', 'rect', 'line', 'open', 'seek', 'fill', 'size', 'turn', 'stop',
+        'home', 'find', 'step', 'tone', 'sqrt', 'RSSI', 'SSID', 'end', 'bit', 'tan',
+        'cos', 'sin', 'pow', 'map', 'abs', 'max', 'min', 'get', 'run', 'put',
+        'isAlphaNumeric', 'isAlpha', 'isAscii', 'isWhitespace', 'isControl', 'isDigit', 
+        'isGraph', 'isLowerCase', 'isPrintable', 'isPunct', 'isSpace', 'isUpperCase', 
+        'isHexadecimalDigit'))    
+
+    # do not highlight
+    suppress_highlight = set((
+        'namespace', 'template', 'mutable', 'using', 'asm', 'typeid', 
+        'typename', 'this', 'alignof', 'constexpr', 'decltype', 'noexcept', 
+        'static_assert', 'thread_local', 'restrict'))
+
+
+    def get_tokens_unprocessed(self, text):
+        for index, token, value in CppLexer.get_tokens_unprocessed(self, text):
+            if value in self.structure:
+                yield index, Name.Builtin, value
+            elif value in self.operators:
+                yield index, Operator, value
+            elif value in self.variables:
+                yield index, Keyword.Reserved, value
+            elif value in self.suppress_highlight:
+                yield index, Name, value
+            elif value in self.functions:
+                yield index, Name.Function, value
+            else:
+                yield index, token, value

packages/wakatime/packages/pygments/lexers/capnproto.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.capnproto
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the Cap'n Proto schema language.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, default
+from pygments.token import Text, Comment, Keyword, Name, Literal
+
+__all__ = ['CapnProtoLexer']
+
+
+class CapnProtoLexer(RegexLexer):
+    """
+    For `Cap'n Proto <https://capnproto.org>`_ source.
+
+    .. versionadded:: 2.2
+    """
+    name = 'Cap\'n Proto'
+    filenames = ['*.capnp']
+    aliases = ['capnp']
+
+    flags = re.MULTILINE | re.UNICODE
+
+    tokens = {
+        'root': [
+            (r'#.*?$', Comment.Single),
+            (r'@[0-9a-zA-Z]*', Name.Decorator),
+            (r'=', Literal, 'expression'),
+            (r':', Name.Class, 'type'),
+            (r'\$', Name.Attribute, 'annotation'),
+            (r'(struct|enum|interface|union|import|using|const|annotation|'
+             r'extends|in|of|on|as|with|from|fixed)\b',
+             Keyword),
+            (r'[\w.]+', Name),
+            (r'[^#@=:$\w]+', Text),
+        ],
+        'type': [
+            (r'[^][=;,(){}$]+', Name.Class),
+            (r'[[(]', Name.Class, 'parentype'),
+            default('#pop'),
+        ],
+        'parentype': [
+            (r'[^][;()]+', Name.Class),
+            (r'[[(]', Name.Class, '#push'),
+            (r'[])]', Name.Class, '#pop'),
+            default('#pop'),
+        ],
+        'expression': [
+            (r'[^][;,(){}$]+', Literal),
+            (r'[[(]', Literal, 'parenexp'),
+            default('#pop'),
+        ],
+        'parenexp': [
+            (r'[^][;()]+', Literal),
+            (r'[[(]', Literal, '#push'),
+            (r'[])]', Literal, '#pop'),
+            default('#pop'),
+        ],
+        'annotation': [
+            (r'[^][;,(){}=:]+', Name.Attribute),
+            (r'[[(]', Name.Attribute, 'annexp'),
+            default('#pop'),
+        ],
+        'annexp': [
+            (r'[^][;()]+', Name.Attribute),
+            (r'[[(]', Name.Attribute, '#push'),
+            (r'[])]', Name.Attribute, '#pop'),
+            default('#pop'),
+        ],
+    }

packages/wakatime/packages/pygments/lexers/chapel.py

@@ -5,7 +5,7 @@
 
     Lexer for the Chapel language.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -42,15 +42,17 @@ class ChapelLexer(RegexLexer):
             (r'(bool|complex|imag|int|opaque|range|real|string|uint)\b',
              Keyword.Type),
             (words((
-                'align', 'atomic', 'begin', 'break', 'by', 'cobegin', 'coforall',
-                'continue', 'delete', 'dmapped', 'do', 'domain', 'else', 'enum',
-                'export', 'extern', 'for', 'forall', 'if', 'index', 'inline',
-                'iter', 'label', 'lambda', 'let', 'local', 'new', 'noinit', 'on',
-                'otherwise', 'pragma', 'reduce', 'return', 'scan', 'select',
-                'serial', 'single', 'sparse', 'subdomain', 'sync', 'then', 'use',
-                'when', 'where', 'while', 'with', 'yield', 'zip'), suffix=r'\b'),
+                'align', 'as', 'atomic', 'begin', 'break', 'by', 'cobegin',
+                'coforall', 'continue', 'delete', 'dmapped', 'do', 'domain',
+                'else', 'enum', 'except', 'export', 'extern', 'for', 'forall',
+                'if', 'index', 'inline', 'iter', 'label', 'lambda', 'let',
+                'local', 'new', 'noinit', 'on', 'only', 'otherwise', 'pragma',
+                'private', 'public', 'reduce', 'require', 'return', 'scan',
+                'select', 'serial', 'single', 'sparse', 'subdomain', 'sync',
+                'then', 'use', 'when', 'where', 'while', 'with', 'yield',
+                'zip'), suffix=r'\b'),
              Keyword),
-            (r'(proc)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'procname'),
+            (r'(proc)((?:\s)+)', bygroups(Keyword, Text), 'procname'),
             (r'(class|module|record|union)(\s+)', bygroups(Keyword, Text),
              'classname'),
 
@@ -76,7 +78,8 @@ class ChapelLexer(RegexLexer):
             (r'[0-9]+', Number.Integer),
 
             # strings
-            (r'["\'](\\\\|\\"|[^"\'])*["\']', String),
+            (r'"(\\\\|\\"|[^"])*"', String),
+            (r"'(\\\\|\\'|[^'])*'", String),
 
             # tokens
             (r'(=|\+=|-=|\*=|/=|\*\*=|%=|&=|\|=|\^=|&&=|\|\|=|<<=|>>=|'
@@ -93,6 +96,7 @@ class ChapelLexer(RegexLexer):
             (r'[a-zA-Z_][\w$]*', Name.Class, '#pop'),
         ],
         'procname': [
-            (r'[a-zA-Z_][\w$]*', Name.Function, '#pop'),
+            (r'([a-zA-Z_][\w$]+|\~[a-zA-Z_][\w$]+|[+*/!~%<>=&^|\-]{1,2})',
+             Name.Function, '#pop'),
         ],
     }

packages/wakatime/packages/pygments/lexers/clean.py

@@ -0,0 +1,288 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.clean
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for the Clean language.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import ExtendedRegexLexer, LexerContext, \
+    bygroups, words, include, default
+from pygments.token import Comment, Keyword, Literal, Name, Number, Operator, \
+    Punctuation, String, Text, Whitespace
+
+__all__ = ['CleanLexer']
+
+
+class CleanLexer(ExtendedRegexLexer):
+    """
+    Lexer for the general purpose, state-of-the-art, pure and lazy functional
+    programming language Clean (http://clean.cs.ru.nl/Clean).
+
+    .. versionadded: 2.2
+    """
+    name = 'Clean'
+    aliases = ['clean']
+    filenames = ['*.icl', '*.dcl']
+
+    def get_tokens_unprocessed(self, text=None, context=None):
+        ctx = LexerContext(text, 0)
+        ctx.indent = 0
+        return ExtendedRegexLexer.get_tokens_unprocessed(self, text, context=ctx)
+
+    def check_class_not_import(lexer, match, ctx):
+        if match.group(0) == 'import':
+            yield match.start(), Keyword.Namespace, match.group(0)
+            ctx.stack = ctx.stack[:-1] + ['fromimportfunc']
+        else:
+            yield match.start(), Name.Class, match.group(0)
+        ctx.pos = match.end()
+
+    def check_instance_class(lexer, match, ctx):
+        if match.group(0) == 'instance' or match.group(0) == 'class':
+            yield match.start(), Keyword, match.group(0)
+        else:
+            yield match.start(), Name.Function, match.group(0)
+            ctx.stack = ctx.stack + ['fromimportfunctype']
+        ctx.pos = match.end()
+
+    @staticmethod
+    def indent_len(text):
+        # Tabs are four spaces:
+        # https://svn.cs.ru.nl/repos/clean-platform/trunk/doc/STANDARDS.txt
+        text = text.replace('\n', '')
+        return len(text.replace('\t', '    ')), len(text)
+
+    def store_indent(lexer, match, ctx):
+        ctx.indent, _ = CleanLexer.indent_len(match.group(0))
+        ctx.pos = match.end()
+        yield match.start(), Text, match.group(0)
+
+    def check_indent1(lexer, match, ctx):
+        indent, reallen = CleanLexer.indent_len(match.group(0))
+        if indent > ctx.indent:
+            yield match.start(), Whitespace, match.group(0)
+            ctx.pos = match.start() + reallen + 1
+        else:
+            ctx.indent = 0
+            ctx.pos = match.start()
+            ctx.stack = ctx.stack[:-1]
+            yield match.start(), Whitespace, match.group(0)[1:]
+
+    def check_indent2(lexer, match, ctx):
+        indent, reallen = CleanLexer.indent_len(match.group(0))
+        if indent > ctx.indent:
+            yield match.start(), Whitespace, match.group(0)
+            ctx.pos = match.start() + reallen + 1
+        else:
+            ctx.indent = 0
+            ctx.pos = match.start()
+            ctx.stack = ctx.stack[:-2]
+
+    def check_indent3(lexer, match, ctx):
+        indent, reallen = CleanLexer.indent_len(match.group(0))
+        if indent > ctx.indent:
+            yield match.start(), Whitespace, match.group(0)
+            ctx.pos = match.start() + reallen + 1
+        else:
+            ctx.indent = 0
+            ctx.pos = match.start()
+            ctx.stack = ctx.stack[:-3]
+            yield match.start(), Whitespace, match.group(0)[1:]
+            if match.group(0) == '\n\n':
+                ctx.pos = ctx.pos + 1
+
+    def skip(lexer, match, ctx):
+        ctx.stack = ctx.stack[:-1]
+        ctx.pos = match.end()
+        yield match.start(), Comment, match.group(0)
+
+    keywords = ('class', 'instance', 'where', 'with', 'let', 'let!',
+                'in', 'case', 'of', 'infix', 'infixr', 'infixl', 'generic',
+                'derive', 'otherwise', 'code', 'inline')
+
+    tokens = {
+        'common': [
+            (r';', Punctuation, '#pop'),
+            (r'//', Comment, 'singlecomment'),
+        ],
+        'root': [
+            # Comments
+            (r'//.*\n', Comment.Single),
+            (r'(?s)/\*\*.*?\*/', Comment.Special),
+            (r'(?s)/\*.*?\*/', Comment.Multi),
+
+            # Modules, imports, etc.
+            (r'\b((?:implementation|definition|system)\s+)?(module)(\s+)([\w`.]+)',
+                bygroups(Keyword.Namespace, Keyword.Namespace, Text, Name.Class)),
+            (r'(?<=\n)import(?=\s)', Keyword.Namespace, 'import'),
+            (r'(?<=\n)from(?=\s)', Keyword.Namespace, 'fromimport'),
+
+            # Keywords
+            # We cannot use (?s)^|(?<=\s) as prefix, so need to repeat this
+            (words(keywords, prefix=r'(?<=\s)', suffix=r'(?=\s)'), Keyword),
+            (words(keywords, prefix=r'^', suffix=r'(?=\s)'), Keyword),
+
+            # Function definitions
+            (r'(?=\{\|)', Whitespace, 'genericfunction'),
+            (r'(?<=\n)([ \t]*)([\w`$()=\-<>~*\^|+&%]+)((?:\s+\w)*)(\s*)(::)',
+             bygroups(store_indent, Name.Function, Keyword.Type, Whitespace,
+                      Punctuation),
+             'functiondefargs'),
+
+            # Type definitions
+            (r'(?<=\n)([ \t]*)(::)', bygroups(store_indent, Punctuation), 'typedef'),
+            (r'^([ \t]*)(::)', bygroups(store_indent, Punctuation), 'typedef'),
+
+            # Literals
+            (r'\'\\?.(?<!\\)\'', String.Char),
+            (r'\'\\\d+\'', String.Char),
+            (r'\'\\\\\'', String.Char),  # (special case for '\\')
+            (r'[+\-~]?\s*\d+\.\d+(E[+\-~]?\d+)?\b', Number.Float),
+            (r'[+\-~]?\s*0[0-7]\b', Number.Oct),
+            (r'[+\-~]?\s*0x[0-9a-fA-F]\b', Number.Hex),
+            (r'[+\-~]?\s*\d+\b', Number.Integer),
+            (r'"', String.Double, 'doubleqstring'),
+            (words(('True', 'False'), prefix=r'(?<=\s)', suffix=r'(?=\s)'),
+             Literal),
+
+            # Qualified names
+            (r'(\')([\w.]+)(\'\.)',
+                bygroups(Punctuation, Name.Namespace, Punctuation)),
+
+            # Everything else is some name
+            (r'([\w`$%/?@]+\.?)*[\w`$%/?@]+', Name),
+
+            # Punctuation
+            (r'[{}()\[\],:;.#]', Punctuation),
+            (r'[+\-=!<>|&~*\^/]', Operator),
+            (r'\\\\', Operator),
+
+            # Lambda expressions
+            (r'\\.*?(->|\.|=)', Name.Function),
+
+            # Whitespace
+            (r'\s', Whitespace),
+
+            include('common'),
+        ],
+        'fromimport': [
+            include('common'),
+            (r'([\w`.]+)', check_class_not_import),
+            (r'\n', Whitespace, '#pop'),
+            (r'\s', Whitespace),
+        ],
+        'fromimportfunc': [
+            include('common'),
+            (r'(::)(\s+)([^,\s]+)', bygroups(Punctuation, Text, Keyword.Type)),
+            (r'([\w`$()=\-<>~*\^|+&%/]+)', check_instance_class),
+            (r',', Punctuation),
+            (r'\n', Whitespace, '#pop'),
+            (r'\s', Whitespace),
+        ],
+        'fromimportfunctype': [
+            include('common'),
+            (r'[{(\[]', Punctuation, 'combtype'),
+            (r',', Punctuation, '#pop'),
+            (r'[:;.#]', Punctuation),
+            (r'\n', Whitespace, '#pop:2'),
+            (r'[^\S\n]+', Whitespace),
+            (r'\S+', Keyword.Type),
+        ],
+        'combtype': [
+            include('common'),
+            (r'[})\]]', Punctuation, '#pop'),
+            (r'[{(\[]', Punctuation, '#pop'),
+            (r'[,:;.#]', Punctuation),
+            (r'\s+', Whitespace),
+            (r'\S+', Keyword.Type),
+        ],
+        'import': [
+            include('common'),
+            (words(('from', 'import', 'as', 'qualified'),
+                   prefix='(?<=\s)', suffix='(?=\s)'), Keyword.Namespace),
+            (r'[\w`.]+', Name.Class),
+            (r'\n', Whitespace, '#pop'),
+            (r',', Punctuation),
+            (r'[^\S\n]+', Whitespace),
+        ],
+        'singlecomment': [
+            (r'(.)(?=\n)', skip),
+            (r'.+(?!\n)', Comment),
+        ],
+        'doubleqstring': [
+            (r'[^\\"]+', String.Double),
+            (r'"', String.Double, '#pop'),
+            (r'\\.', String.Double),
+        ],
+        'typedef': [
+            include('common'),
+            (r'[\w`]+', Keyword.Type),
+            (r'[:=|(),\[\]{}!*]', Punctuation),
+            (r'->', Punctuation),
+            (r'\n(?=[^\s|])', Whitespace, '#pop'),
+            (r'\s', Whitespace),
+            (r'.', Keyword.Type),
+        ],
+        'genericfunction': [
+            include('common'),
+            (r'\{\|', Punctuation),
+            (r'\|\}', Punctuation, '#pop'),
+            (r',', Punctuation),
+            (r'->', Punctuation),
+            (r'(\s+of\s+)(\{)', bygroups(Keyword, Punctuation), 'genericftypes'),
+            (r'\s', Whitespace),
+            (r'[\w`\[\]{}!]+', Keyword.Type),
+            (r'[*()]', Punctuation),
+        ],
+        'genericftypes': [
+            include('common'),
+            (r'[\w`]+', Keyword.Type),
+            (r',', Punctuation),
+            (r'\s', Whitespace),
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'functiondefargs': [
+            include('common'),
+            (r'\n(\s*)', check_indent1),
+            (r'[!{}()\[\],:;.#]', Punctuation),
+            (r'->', Punctuation, 'functiondefres'),
+            (r'^(?=\S)', Whitespace, '#pop'),
+            (r'\S', Keyword.Type),
+            (r'\s', Whitespace),
+        ],
+        'functiondefres': [
+            include('common'),
+            (r'\n(\s*)', check_indent2),
+            (r'^(?=\S)', Whitespace, '#pop:2'),
+            (r'[!{}()\[\],:;.#]', Punctuation),
+            (r'\|', Punctuation, 'functiondefclasses'),
+            (r'\S', Keyword.Type),
+            (r'\s', Whitespace),
+        ],
+        'functiondefclasses': [
+            include('common'),
+            (r'\n(\s*)', check_indent3),
+            (r'^(?=\S)', Whitespace, '#pop:3'),
+            (r'[,&]', Punctuation),
+            (r'\[', Punctuation, 'functiondefuniquneq'),
+            (r'[\w`$()=\-<>~*\^|+&%/{}\[\]@]', Name.Function, 'functionname'),
+            (r'\s+', Whitespace),
+        ],
+        'functiondefuniquneq': [
+            include('common'),
+            (r'[a-z]+', Keyword.Type),
+            (r'\s+', Whitespace),
+            (r'<=|,', Punctuation),
+            (r'\]', Punctuation, '#pop')
+        ],
+        'functionname': [
+            include('common'),
+            (r'[\w`$()=\-<>~*\^|+&%/]+', Name.Function),
+            (r'(?=\{\|)', Punctuation, 'genericfunction'),
+            default('#pop'),
+        ]
+    }

packages/wakatime/packages/pygments/lexers/compiled.py

@@ -5,7 +5,7 @@
 
     Just export lexer classes previously contained in this module.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -29,5 +29,6 @@ from pygments.lexers.dylan import DylanLexer, DylanLidLexer, DylanConsoleLexer
 from pygments.lexers.ooc import OocLexer
 from pygments.lexers.felix import FelixLexer
 from pygments.lexers.nimrod import NimrodLexer
+from pygments.lexers.crystal import CrystalLexer
 
 __all__ = []

packages/wakatime/packages/pygments/lexers/configs.py

@@ -5,7 +5,7 @@
 
     Lexers for configuration file formats.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -13,12 +13,14 @@ import re
 
 from pygments.lexer import RegexLexer, default, words, bygroups, include, using
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
-    Number, Punctuation, Whitespace
+    Number, Punctuation, Whitespace, Literal
 from pygments.lexers.shell import BashLexer
 
 __all__ = ['IniLexer', 'RegeditLexer', 'PropertiesLexer', 'KconfigLexer',
            'Cfengine3Lexer', 'ApacheConfLexer', 'SquidConfLexer',
-           'NginxConfLexer', 'LighttpdConfLexer', 'DockerLexer']
+           'NginxConfLexer', 'LighttpdConfLexer', 'DockerLexer',
+           'TerraformLexer', 'TermcapLexer', 'TerminfoLexer',
+           'PkgConfigLexer', 'PacmanConfLexer']
 
 
 class IniLexer(RegexLexer):
@@ -28,8 +30,8 @@ class IniLexer(RegexLexer):
 
     name = 'INI'
     aliases = ['ini', 'cfg', 'dosini']
-    filenames = ['*.ini', '*.cfg']
-    mimetypes = ['text/x-ini']
+    filenames = ['*.ini', '*.cfg', '*.inf']
+    mimetypes = ['text/x-ini', 'text/inf']
 
     tokens = {
         'root': [
@@ -37,8 +39,10 @@ class IniLexer(RegexLexer):
             (r'[;#].*', Comment.Single),
             (r'\[.*?\]$', Keyword),
             (r'(.*?)([ \t]*)(=)([ \t]*)(.*(?:\n[ \t].+)*)',
-             bygroups(Name.Attribute, Text, Operator, Text, String))
-        ]
+             bygroups(Name.Attribute, Text, Operator, Text, String)),
+            # standalone option, supported by some INI parsers
+            (r'(.+?)$', Name.Attribute),
+        ],
     }
 
     def analyse_text(text):
@@ -96,6 +100,8 @@ class PropertiesLexer(RegexLexer):
     """
     Lexer for configuration files in Java's properties format.
 
+    Note: trailing whitespace counts as part of the value as per spec
+
     .. versionadded:: 1.4
     """
 
@@ -106,10 +112,14 @@ class PropertiesLexer(RegexLexer):
 
     tokens = {
         'root': [
-            (r'\s+', Text),
-            (r'(?:[;#]|//).*$', Comment),
+            (r'^(\w+)([ \t])(\w+\s*)$', bygroups(Name.Attribute, Text, String)),
+            (r'^\w+(\\[ \t]\w*)*$', Name.Attribute),
+            (r'(^ *)([#!].*)', bygroups(Text, Comment)),
+            # More controversial comments
+            (r'(^ *)((?:;|//).*)', bygroups(Text, Comment)),
             (r'(.*?)([ \t]*)([=:])([ \t]*)(.*(?:(?<=\\)\n.*)*)',
              bygroups(Name.Attribute, Text, Operator, Text, String)),
+            (r'\s', Text),
         ],
     }
 
@@ -452,7 +462,7 @@ class NginxConfLexer(RegexLexer):
     """
     name = 'Nginx configuration file'
     aliases = ['nginx']
-    filenames = []
+    filenames = ['nginx.conf']
     mimetypes = ['text/x-nginx-conf']
 
     tokens = {
@@ -540,7 +550,284 @@ class DockerLexer(RegexLexer):
              bygroups(Name.Keyword, Whitespace, Keyword)),
             (r'^(%s)\b(.*)' % (_keywords,), bygroups(Keyword, String)),
             (r'#.*', Comment),
-            (r'RUN', Keyword), # Rest of line falls through
+            (r'RUN', Keyword),  # Rest of line falls through
             (r'(.*\\\n)*.+', using(BashLexer)),
         ],
     }
+
+
+class TerraformLexer(RegexLexer):
+    """
+    Lexer for `terraformi .tf files <https://www.terraform.io/>`_.
+
+    .. versionadded:: 2.1
+    """
+
+    name = 'Terraform'
+    aliases = ['terraform', 'tf']
+    filenames = ['*.tf']
+    mimetypes = ['application/x-tf', 'application/x-terraform']
+
+    tokens = {
+        'root': [
+             include('string'),
+             include('punctuation'),
+             include('curly'),
+             include('basic'),
+             include('whitespace'),
+             (r'[0-9]+', Number),
+        ],
+        'basic': [
+             (words(('true', 'false'), prefix=r'\b', suffix=r'\b'), Keyword.Type),
+             (r'\s*/\*', Comment.Multiline, 'comment'),
+             (r'\s*#.*\n', Comment.Single),
+             (r'(.*?)(\s*)(=)', bygroups(Name.Attribute, Text, Operator)),
+             (words(('variable', 'resource', 'provider', 'provisioner', 'module'),
+                    prefix=r'\b', suffix=r'\b'), Keyword.Reserved, 'function'),
+             (words(('ingress', 'egress', 'listener', 'default', 'connection'),
+                    prefix=r'\b', suffix=r'\b'), Keyword.Declaration),
+             ('\$\{', String.Interpol, 'var_builtin'),
+        ],
+        'function': [
+             (r'(\s+)(".*")(\s+)', bygroups(Text, String, Text)),
+             include('punctuation'),
+             include('curly'),
+        ],
+        'var_builtin': [
+            (r'\$\{', String.Interpol, '#push'),
+            (words(('concat', 'file', 'join', 'lookup', 'element'),
+                   prefix=r'\b', suffix=r'\b'), Name.Builtin),
+            include('string'),
+            include('punctuation'),
+            (r'\s+', Text),
+            (r'\}', String.Interpol, '#pop'),
+        ],
+        'string': [
+            (r'(".*")', bygroups(String.Double)),
+        ],
+        'punctuation': [
+            (r'[\[\](),.]', Punctuation),
+        ],
+        # Keep this seperate from punctuation - we sometimes want to use different
+        # Tokens for { }
+        'curly': [
+            (r'\{', Text.Punctuation),
+            (r'\}', Text.Punctuation),
+        ],
+        'comment': [
+            (r'[^*/]', Comment.Multiline),
+            (r'/\*', Comment.Multiline, '#push'),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'[*/]', Comment.Multiline)
+        ],
+        'whitespace': [
+            (r'\n', Text),
+            (r'\s+', Text),
+            (r'\\\n', Text),
+        ],
+    }
+
+
+class TermcapLexer(RegexLexer):
+    """
+    Lexer for termcap database source.
+
+    This is very simple and minimal.
+
+    .. versionadded:: 2.1
+    """
+    name = 'Termcap'
+    aliases = ['termcap']
+    filenames = ['termcap', 'termcap.src']
+    mimetypes = []
+
+    # NOTE:
+    #   * multiline with trailing backslash
+    #   * separator is ':'
+    #   * to embed colon as data, we must use \072
+    #   * space after separator is not allowed (mayve)
+    tokens = {
+        'root': [
+            (r'^#.*$', Comment),
+            (r'^[^\s#:|]+', Name.Tag, 'names'),
+        ],
+        'names': [
+            (r'\n', Text, '#pop'),
+            (r':', Punctuation, 'defs'),
+            (r'\|', Punctuation),
+            (r'[^:|]+', Name.Attribute),
+        ],
+        'defs': [
+            (r'\\\n[ \t]*', Text),
+            (r'\n[ \t]*', Text, '#pop:2'),
+            (r'(#)([0-9]+)', bygroups(Operator, Number)),
+            (r'=', Operator, 'data'),
+            (r':', Punctuation),
+            (r'[^\s:=#]+', Name.Class),
+        ],
+        'data': [
+            (r'\\072', Literal),
+            (r':', Punctuation, '#pop'),
+            (r'[^:\\]+', Literal),  # for performance
+            (r'.', Literal),
+        ],
+    }
+
+
+class TerminfoLexer(RegexLexer):
+    """
+    Lexer for terminfo database source.
+
+    This is very simple and minimal.
+
+    .. versionadded:: 2.1
+    """
+    name = 'Terminfo'
+    aliases = ['terminfo']
+    filenames = ['terminfo', 'terminfo.src']
+    mimetypes = []
+
+    # NOTE:
+    #   * multiline with leading whitespace
+    #   * separator is ','
+    #   * to embed comma as data, we can use \,
+    #   * space after separator is allowed
+    tokens = {
+        'root': [
+            (r'^#.*$', Comment),
+            (r'^[^\s#,|]+', Name.Tag, 'names'),
+        ],
+        'names': [
+            (r'\n', Text, '#pop'),
+            (r'(,)([ \t]*)', bygroups(Punctuation, Text), 'defs'),
+            (r'\|', Punctuation),
+            (r'[^,|]+', Name.Attribute),
+        ],
+        'defs': [
+            (r'\n[ \t]+', Text),
+            (r'\n', Text, '#pop:2'),
+            (r'(#)([0-9]+)', bygroups(Operator, Number)),
+            (r'=', Operator, 'data'),
+            (r'(,)([ \t]*)', bygroups(Punctuation, Text)),
+            (r'[^\s,=#]+', Name.Class),
+        ],
+        'data': [
+            (r'\\[,\\]', Literal),
+            (r'(,)([ \t]*)', bygroups(Punctuation, Text), '#pop'),
+            (r'[^\\,]+', Literal),  # for performance
+            (r'.', Literal),
+        ],
+    }
+
+
+class PkgConfigLexer(RegexLexer):
+    """
+    Lexer for `pkg-config
+    <http://www.freedesktop.org/wiki/Software/pkg-config/>`_
+    (see also `manual page <http://linux.die.net/man/1/pkg-config>`_).
+
+    .. versionadded:: 2.1
+    """
+
+    name = 'PkgConfig'
+    aliases = ['pkgconfig']
+    filenames = ['*.pc']
+    mimetypes = []
+
+    tokens = {
+        'root': [
+            (r'#.*$', Comment.Single),
+
+            # variable definitions
+            (r'^(\w+)(=)', bygroups(Name.Attribute, Operator)),
+
+            # keyword lines
+            (r'^([\w.]+)(:)',
+             bygroups(Name.Tag, Punctuation), 'spvalue'),
+
+            # variable references
+            include('interp'),
+
+            # fallback
+            (r'[^${}#=:\n.]+', Text),
+            (r'.', Text),
+        ],
+        'interp': [
+            # you can escape literal "$" as "$$"
+            (r'\$\$', Text),
+
+            # variable references
+            (r'\$\{', String.Interpol, 'curly'),
+        ],
+        'curly': [
+            (r'\}', String.Interpol, '#pop'),
+            (r'\w+', Name.Attribute),
+        ],
+        'spvalue': [
+            include('interp'),
+
+            (r'#.*$', Comment.Single, '#pop'),
+            (r'\n', Text, '#pop'),
+
+            # fallback
+            (r'[^${}#\n]+', Text),
+            (r'.', Text),
+        ],
+    }
+
+
+class PacmanConfLexer(RegexLexer):
+    """
+    Lexer for `pacman.conf
+    <https://www.archlinux.org/pacman/pacman.conf.5.html>`_.
+
+    Actually, IniLexer works almost fine for this format,
+    but it yield error token. It is because pacman.conf has
+    a form without assignment like:
+
+        UseSyslog
+        Color
+        TotalDownload
+        CheckSpace
+        VerbosePkgLists
+
+    These are flags to switch on.
+
+    .. versionadded:: 2.1
+    """
+
+    name = 'PacmanConf'
+    aliases = ['pacmanconf']
+    filenames = ['pacman.conf']
+    mimetypes = []
+
+    tokens = {
+        'root': [
+            # comment
+            (r'#.*$', Comment.Single),
+
+            # section header
+            (r'^\s*\[.*?\]\s*$', Keyword),
+
+            # variable definitions
+            # (Leading space is allowed...)
+            (r'(\w+)(\s*)(=)',
+             bygroups(Name.Attribute, Text, Operator)),
+
+            # flags to on
+            (r'^(\s*)(\w+)(\s*)$',
+             bygroups(Text, Name.Attribute, Text)),
+
+            # built-in special values
+            (words((
+                '$repo',  # repository
+                '$arch',  # architecture
+                '%o',     # outfile
+                '%u',     # url
+                ), suffix=r'\b'),
+             Name.Variable),
+
+            # fallback
+            (r'.', Text),
+        ],
+    }

packages/wakatime/packages/pygments/lexers/console.py

@@ -5,7 +5,7 @@
 
     Lexers for misc console output.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -18,7 +18,7 @@ __all__ = ['VCTreeStatusLexer', 'PyPyLogLexer']
 
 class VCTreeStatusLexer(RegexLexer):
     """
-    For colorizing output of version control status commans, like "hg
+    For colorizing output of version control status commands, like "hg
     status" or "svn status".
 
     .. versionadded:: 2.0

packages/wakatime/packages/pygments/lexers/crystal.py

@@ -0,0 +1,393 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.crystal
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Crystal.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import ExtendedRegexLexer, include, \
+    bygroups, default, LexerContext, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Error
+
+__all__ = ['CrystalLexer']
+
+line_re = re.compile('.*?\n')
+
+
+CRYSTAL_OPERATORS = [
+    '!=', '!~', '!', '%', '&&', '&', '**', '*', '+', '-', '/', '<=>', '<<', '<=', '<',
+    '===', '==', '=~', '=', '>=', '>>', '>', '[]=', '[]?', '[]', '^', '||', '|', '~'
+]
+
+
+class CrystalLexer(ExtendedRegexLexer):
+    """
+    For `Crystal <http://crystal-lang.org>`_ source code.
+
+    .. versionadded:: 2.2
+    """
+
+    name = 'Crystal'
+    aliases = ['cr', 'crystal']
+    filenames = ['*.cr']
+    mimetypes = ['text/x-crystal']
+
+    flags = re.DOTALL | re.MULTILINE
+
+    def heredoc_callback(self, match, ctx):
+        # okay, this is the hardest part of parsing Crystal...
+        # match: 1 = <<-?, 2 = quote? 3 = name 4 = quote? 5 = rest of line
+
+        start = match.start(1)
+        yield start, Operator, match.group(1)        # <<-?
+        yield match.start(2), String.Heredoc, match.group(2)    # quote ", ', `
+        yield match.start(3), String.Delimiter, match.group(3)  # heredoc name
+        yield match.start(4), String.Heredoc, match.group(4)    # quote again
+
+        heredocstack = ctx.__dict__.setdefault('heredocstack', [])
+        outermost = not bool(heredocstack)
+        heredocstack.append((match.group(1) == '<<-', match.group(3)))
+
+        ctx.pos = match.start(5)
+        ctx.end = match.end(5)
+        # this may find other heredocs
+        for i, t, v in self.get_tokens_unprocessed(context=ctx):
+            yield i, t, v
+        ctx.pos = match.end()
+
+        if outermost:
+            # this is the outer heredoc again, now we can process them all
+            for tolerant, hdname in heredocstack:
+                lines = []
+                for match in line_re.finditer(ctx.text, ctx.pos):
+                    if tolerant:
+                        check = match.group().strip()
+                    else:
+                        check = match.group().rstrip()
+                    if check == hdname:
+                        for amatch in lines:
+                            yield amatch.start(), String.Heredoc, amatch.group()
+                        yield match.start(), String.Delimiter, match.group()
+                        ctx.pos = match.end()
+                        break
+                    else:
+                        lines.append(match)
+                else:
+                    # end of heredoc not found -- error!
+                    for amatch in lines:
+                        yield amatch.start(), Error, amatch.group()
+            ctx.end = len(ctx.text)
+            del heredocstack[:]
+
+    def gen_crystalstrings_rules():
+        def intp_regex_callback(self, match, ctx):
+            yield match.start(1), String.Regex, match.group(1)  # begin
+            nctx = LexerContext(match.group(3), 0, ['interpolated-regex'])
+            for i, t, v in self.get_tokens_unprocessed(context=nctx):
+                yield match.start(3)+i, t, v
+            yield match.start(4), String.Regex, match.group(4)  # end[imsx]*
+            ctx.pos = match.end()
+
+        def intp_string_callback(self, match, ctx):
+            yield match.start(1), String.Other, match.group(1)
+            nctx = LexerContext(match.group(3), 0, ['interpolated-string'])
+            for i, t, v in self.get_tokens_unprocessed(context=nctx):
+                yield match.start(3)+i, t, v
+            yield match.start(4), String.Other, match.group(4)  # end
+            ctx.pos = match.end()
+
+        states = {}
+        states['strings'] = [
+            (r'\:@{0,2}[a-zA-Z_]\w*[!?]?', String.Symbol),
+            (words(CRYSTAL_OPERATORS, prefix=r'\:@{0,2}'), String.Symbol),
+            (r":'(\\\\|\\'|[^'])*'", String.Symbol),
+            # This allows arbitrary text after '\ for simplicity
+            (r"'(\\\\|\\'|[^']|\\[^'\\]+)'", String.Char),
+            (r':"', String.Symbol, 'simple-sym'),
+            # Crystal doesn't have "symbol:"s but this simplifies function args
+            (r'([a-zA-Z_]\w*)(:)(?!:)', bygroups(String.Symbol, Punctuation)),
+            (r'"', String.Double, 'simple-string'),
+            (r'(?<!\.)`', String.Backtick, 'simple-backtick'),
+        ]
+
+        # double-quoted string and symbol
+        for name, ttype, end in ('string', String.Double, '"'), \
+                                ('sym', String.Symbol, '"'), \
+                                ('backtick', String.Backtick, '`'):
+            states['simple-'+name] = [
+                include('string-escaped' if name == 'sym' else 'string-intp-escaped'),
+                (r'[^\\%s#]+' % end, ttype),
+                (r'[\\#]', ttype),
+                (end, ttype, '#pop'),
+            ]
+
+        # braced quoted strings
+        for lbrace, rbrace, bracecc, name in \
+                ('\\{', '\\}', '{}', 'cb'), \
+                ('\\[', '\\]', '\\[\\]', 'sb'), \
+                ('\\(', '\\)', '()', 'pa'), \
+                ('<', '>', '<>', 'ab'):
+            states[name+'-intp-string'] = [
+                (r'\\[' + lbrace + ']', String.Other),
+                (lbrace, String.Other, '#push'),
+                (rbrace, String.Other, '#pop'),
+                include('string-intp-escaped'),
+                (r'[\\#' + bracecc + ']', String.Other),
+                (r'[^\\#' + bracecc + ']+', String.Other),
+            ]
+            states['strings'].append((r'%' + lbrace, String.Other,
+                                      name+'-intp-string'))
+            states[name+'-string'] = [
+                (r'\\[\\' + bracecc + ']', String.Other),
+                (lbrace, String.Other, '#push'),
+                (rbrace, String.Other, '#pop'),
+                (r'[\\#' + bracecc + ']', String.Other),
+                (r'[^\\#' + bracecc + ']+', String.Other),
+            ]
+            # http://crystal-lang.org/docs/syntax_and_semantics/literals/array.html
+            states['strings'].append((r'%[wi]' + lbrace, String.Other,
+                                      name+'-string'))
+            states[name+'-regex'] = [
+                (r'\\[\\' + bracecc + ']', String.Regex),
+                (lbrace, String.Regex, '#push'),
+                (rbrace + '[imsx]*', String.Regex, '#pop'),
+                include('string-intp'),
+                (r'[\\#' + bracecc + ']', String.Regex),
+                (r'[^\\#' + bracecc + ']+', String.Regex),
+            ]
+            states['strings'].append((r'%r' + lbrace, String.Regex,
+                                      name+'-regex'))
+
+        # these must come after %<brace>!
+        states['strings'] += [
+            # %r regex
+            (r'(%r([\W_]))((?:\\\2|(?!\2).)*)(\2[imsx]*)',
+             intp_regex_callback),
+            # regular fancy strings with qsw
+            (r'(%[wi]([\W_]))((?:\\\2|(?!\2).)*)(\2)',
+             intp_string_callback),
+            # special forms of fancy strings after operators or
+            # in method calls with braces
+            (r'(?<=[-+/*%=<>&!^|~,(])(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
+             bygroups(Text, String.Other, None)),
+            # and because of fixed width lookbehinds the whole thing a
+            # second time for line startings...
+            (r'^(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
+             bygroups(Text, String.Other, None)),
+            # all regular fancy strings without qsw
+            (r'(%([\[{(<]))((?:\\\2|(?!\2).)*)(\2)',
+             intp_string_callback),
+        ]
+
+        return states
+
+    tokens = {
+        'root': [
+            (r'#.*?$', Comment.Single),
+            # keywords
+            (words('''
+                abstract asm as begin break case do else elsif end ensure extend ifdef if
+                include instance_sizeof next of pointerof private protected rescue return
+                require sizeof super then typeof unless until when while with yield
+            '''.split(), suffix=r'\b'), Keyword),
+            (words(['true', 'false', 'nil'], suffix=r'\b'), Keyword.Constant),
+            # start of function, class and module names
+            (r'(module|lib)(\s+)([a-zA-Z_]\w*(?:::[a-zA-Z_]\w*)*)',
+             bygroups(Keyword, Text, Name.Namespace)),
+            (r'(def|fun|macro)(\s+)((?:[a-zA-Z_]\w*::)*)',
+             bygroups(Keyword, Text, Name.Namespace), 'funcname'),
+            (r'def(?=[*%&^`~+-/\[<>=])', Keyword, 'funcname'),
+            (r'(class|struct|union|type|alias|enum)(\s+)((?:[a-zA-Z_]\w*::)*)',
+             bygroups(Keyword, Text, Name.Namespace), 'classname'),
+            (r'(self|out|uninitialized)\b|(is_a|responds_to)\?', Keyword.Pseudo),
+            # macros
+            (words('''
+                debugger record pp assert_responds_to spawn parallel
+                getter setter property delegate def_hash def_equals def_equals_and_hash
+                forward_missing_to
+            '''.split(), suffix=r'\b'), Name.Builtin.Pseudo),
+            (r'getter[!?]|property[!?]|__(DIR|FILE|LINE)__\b', Name.Builtin.Pseudo),
+            # builtins
+            # http://crystal-lang.org/api/toplevel.html
+            (words('''
+                Object Value Struct Reference Proc Class Nil Symbol Enum Void
+                Bool Number Int Int8 Int16 Int32 Int64 UInt8 UInt16 UInt32 UInt64
+                Float Float32 Float64 Char String
+                Pointer Slice Range Exception Regex
+                Mutex StaticArray Array Hash Set Tuple Deque Box Process File
+                Dir Time Channel Concurrent Scheduler
+                abort at_exit caller delay exit fork future get_stack_top gets
+                lazy loop main p print printf puts
+                raise rand read_line sleep sprintf system with_color
+            '''.split(), prefix=r'(?<!\.)', suffix=r'\b'), Name.Builtin),
+            # normal heredocs
+            (r'(?<!\w)(<<-?)(["`\']?)([a-zA-Z_]\w*)(\2)(.*?\n)',
+             heredoc_callback),
+            # empty string heredocs
+            (r'(<<-?)("|\')()(\2)(.*?\n)', heredoc_callback),
+            (r'__END__', Comment.Preproc, 'end-part'),
+            # multiline regex (after keywords or assignments)
+            (r'(?:^|(?<=[=<>~!:])|'
+             r'(?<=(?:\s|;)when\s)|'
+             r'(?<=(?:\s|;)or\s)|'
+             r'(?<=(?:\s|;)and\s)|'
+             r'(?<=\.index\s)|'
+             r'(?<=\.scan\s)|'
+             r'(?<=\.sub\s)|'
+             r'(?<=\.sub!\s)|'
+             r'(?<=\.gsub\s)|'
+             r'(?<=\.gsub!\s)|'
+             r'(?<=\.match\s)|'
+             r'(?<=(?:\s|;)if\s)|'
+             r'(?<=(?:\s|;)elsif\s)|'
+             r'(?<=^when\s)|'
+             r'(?<=^index\s)|'
+             r'(?<=^scan\s)|'
+             r'(?<=^sub\s)|'
+             r'(?<=^gsub\s)|'
+             r'(?<=^sub!\s)|'
+             r'(?<=^gsub!\s)|'
+             r'(?<=^match\s)|'
+             r'(?<=^if\s)|'
+             r'(?<=^elsif\s)'
+             r')(\s*)(/)', bygroups(Text, String.Regex), 'multiline-regex'),
+            # multiline regex (in method calls or subscripts)
+            (r'(?<=\(|,|\[)/', String.Regex, 'multiline-regex'),
+            # multiline regex (this time the funny no whitespace rule)
+            (r'(\s+)(/)(?![\s=])', bygroups(Text, String.Regex),
+             'multiline-regex'),
+            # lex numbers and ignore following regular expressions which
+            # are division operators in fact (grrrr. i hate that. any
+            # better ideas?)
+            # since pygments 0.7 we also eat a "?" operator after numbers
+            # so that the char operator does not work. Chars are not allowed
+            # there so that you can use the ternary operator.
+            # stupid example:
+            #   x>=0?n[x]:""
+            (r'(0o[0-7]+(?:_[0-7]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?',
+             bygroups(Number.Oct, Text, Operator)),
+            (r'(0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?',
+             bygroups(Number.Hex, Text, Operator)),
+            (r'(0b[01]+(?:_[01]+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?',
+             bygroups(Number.Bin, Text, Operator)),
+            # 3 separate expressions for floats because any of the 3 optional
+            # parts makes it a float
+            (r'((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)(?:e[+-]?[0-9]+)?'
+             r'(?:_?f[0-9]+)?)(\s*)([/?])?',
+             bygroups(Number.Float, Text, Operator)),
+            (r'((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)?(?:e[+-]?[0-9]+)'
+             r'(?:_?f[0-9]+)?)(\s*)([/?])?',
+             bygroups(Number.Float, Text, Operator)),
+            (r'((?:0(?![0-9])|[1-9][\d_]*)(?:\.\d[\d_]*)?(?:e[+-]?[0-9]+)?'
+             r'(?:_?f[0-9]+))(\s*)([/?])?',
+             bygroups(Number.Float, Text, Operator)),
+            (r'(0\b|[1-9][\d]*(?:_\d+)*(?:_?[iu][0-9]+)?)\b(\s*)([/?])?',
+             bygroups(Number.Integer, Text, Operator)),
+            # Names
+            (r'@@[a-zA-Z_]\w*', Name.Variable.Class),
+            (r'@[a-zA-Z_]\w*', Name.Variable.Instance),
+            (r'\$\w+', Name.Variable.Global),
+            (r'\$[!@&`\'+~=/\\,;.<>_*$?:"^-]', Name.Variable.Global),
+            (r'\$-[0adFiIlpvw]', Name.Variable.Global),
+            (r'::', Operator),
+            include('strings'),
+            # chars
+            (r'\?(\\[MC]-)*'  # modifiers
+             r'(\\([\\befnrtv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})|\S)'
+             r'(?!\w)',
+             String.Char),
+            (r'[A-Z][A-Z_]+\b', Name.Constant),
+            # macro expansion
+            (r'\{%', String.Interpol, 'in-macro-control'),
+            (r'\{\{', String.Interpol, 'in-macro-expr'),
+            # attributes
+            (r'(@\[)(\s*)([A-Z]\w*)',
+             bygroups(Operator, Text, Name.Decorator), 'in-attr'),
+            # this is needed because Crystal attributes can look
+            # like keywords (class) or like this: ` ?!?
+            (words(CRYSTAL_OPERATORS, prefix=r'(\.|::)'),
+             bygroups(Operator, Name.Operator)),
+            (r'(\.|::)([a-zA-Z_]\w*[!?]?|[*%&^`~+\-/\[<>=])',
+             bygroups(Operator, Name)),
+            # Names can end with [!?] unless it's "!="
+            (r'[a-zA-Z_]\w*(?:[!?](?!=))?', Name),
+            (r'(\[|\]\??|\*\*|<=>?|>=|<<?|>>?|=~|===|'
+             r'!~|&&?|\|\||\.{1,3})', Operator),
+            (r'[-+/*%=<>&!^|~]=?', Operator),
+            (r'[(){};,/?:\\]', Punctuation),
+            (r'\s+', Text)
+        ],
+        'funcname': [
+            (r'(?:([a-zA-Z_]\w*)(\.))?'
+             r'([a-zA-Z_]\w*[!?]?|\*\*?|[-+]@?|'
+             r'[/%&|^`~]|\[\]=?|<<|>>|<=?>|>=?|===?)',
+             bygroups(Name.Class, Operator, Name.Function), '#pop'),
+            default('#pop')
+        ],
+        'classname': [
+            (r'[A-Z_]\w*', Name.Class),
+            (r'(\()(\s*)([A-Z_]\w*)(\s*)(\))',
+             bygroups(Punctuation, Text, Name.Class, Text, Punctuation)),
+            default('#pop')
+        ],
+        'in-intp': [
+            (r'\{', String.Interpol, '#push'),
+            (r'\}', String.Interpol, '#pop'),
+            include('root'),
+        ],
+        'string-intp': [
+            (r'#\{', String.Interpol, 'in-intp'),
+        ],
+        'string-escaped': [
+            (r'\\([\\befnstv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})', String.Escape)
+        ],
+        'string-intp-escaped': [
+            include('string-intp'),
+            include('string-escaped'),
+        ],
+        'interpolated-regex': [
+            include('string-intp'),
+            (r'[\\#]', String.Regex),
+            (r'[^\\#]+', String.Regex),
+        ],
+        'interpolated-string': [
+            include('string-intp'),
+            (r'[\\#]', String.Other),
+            (r'[^\\#]+', String.Other),
+        ],
+        'multiline-regex': [
+            include('string-intp'),
+            (r'\\\\', String.Regex),
+            (r'\\/', String.Regex),
+            (r'[\\#]', String.Regex),
+            (r'[^\\/#]+', String.Regex),
+            (r'/[imsx]*', String.Regex, '#pop'),
+        ],
+        'end-part': [
+            (r'.+', Comment.Preproc, '#pop')
+        ],
+        'in-macro-control': [
+            (r'\{%', String.Interpol, '#push'),
+            (r'%\}', String.Interpol, '#pop'),
+            (r'for\b|in\b', Keyword),
+            include('root'),
+        ],
+        'in-macro-expr': [
+            (r'\{\{', String.Interpol, '#push'),
+            (r'\}\}', String.Interpol, '#pop'),
+            include('root'),
+        ],
+        'in-attr': [
+            (r'\[', Operator, '#push'),
+            (r'\]', Operator, '#pop'),
+            include('root'),
+        ],
+    }
+    tokens.update(gen_crystalstrings_rules())

packages/wakatime/packages/pygments/lexers/csound.py

@@ -0,0 +1,366 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.csound
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for CSound languages.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, default, include, using, words
+from pygments.token import Comment, Keyword, Name, Number, Operator, Punctuation, \
+    String, Text
+from pygments.lexers._csound_builtins import OPCODES
+from pygments.lexers.html import HtmlLexer
+from pygments.lexers.python import PythonLexer
+from pygments.lexers.scripting import LuaLexer
+
+__all__ = ['CsoundScoreLexer', 'CsoundOrchestraLexer', 'CsoundDocumentLexer']
+
+newline = (r'((?:(?:;|//).*)*)(\n)', bygroups(Comment.Single, Text))
+
+
+class CsoundLexer(RegexLexer):
+    # Subclasses must define a 'single-line string' state.
+    tokens = {
+        'whitespace': [
+            (r'[ \t]+', Text),
+            (r'\\\n', Text),
+            (r'/[*](.|\n)*?[*]/', Comment.Multiline)
+        ],
+
+        'macro call': [
+            (r'(\$\w+\.?)(\()', bygroups(Comment.Preproc, Punctuation),
+             'function macro call'),
+            (r'\$\w+(\.|\b)', Comment.Preproc)
+        ],
+        'function macro call': [
+            (r"((?:\\['\)]|[^'\)])+)(')", bygroups(Comment.Preproc, Punctuation)),
+            (r"([^'\)]+)(\))", bygroups(Comment.Preproc, Punctuation), '#pop')
+        ],
+
+        'whitespace or macro call': [
+            include('whitespace'),
+            include('macro call')
+        ],
+
+        'preprocessor directives': [
+            (r'#(e(nd(if)?|lse)|ifn?def|undef)\b|##', Comment.Preproc),
+            (r'#include\b', Comment.Preproc, 'include'),
+            (r'#[ \t]*define\b', Comment.Preproc, 'macro name'),
+            (r'@+[ \t]*\d*', Comment.Preproc)
+        ],
+
+        'include': [
+            include('whitespace'),
+            (r'"', String, 'single-line string')
+        ],
+
+        'macro name': [
+            include('whitespace'),
+            (r'(\w+)(\()', bygroups(Comment.Preproc, Text),
+             'function macro argument list'),
+            (r'\w+', Comment.Preproc, 'object macro definition after name')
+        ],
+        'object macro definition after name': [
+            include('whitespace'),
+            (r'#', Punctuation, 'object macro replacement text')
+        ],
+        'object macro replacement text': [
+            (r'(\\#|[^#])+', Comment.Preproc),
+            (r'#', Punctuation, '#pop:3')
+        ],
+        'function macro argument list': [
+            (r"(\w+)(['#])", bygroups(Comment.Preproc, Punctuation)),
+            (r'(\w+)(\))', bygroups(Comment.Preproc, Punctuation),
+             'function macro definition after name')
+        ],
+        'function macro definition after name': [
+            (r'[ \t]+', Text),
+            (r'#', Punctuation, 'function macro replacement text')
+        ],
+        'function macro replacement text': [
+            (r'(\\#|[^#])+', Comment.Preproc),
+            (r'#', Punctuation, '#pop:4')
+        ]
+    }
+
+
+class CsoundScoreLexer(CsoundLexer):
+    """
+    For `Csound <http://csound.github.io>`_ scores.
+
+    .. versionadded:: 2.1
+    """
+
+    name = 'Csound Score'
+    aliases = ['csound-score', 'csound-sco']
+    filenames = ['*.sco']
+
+    tokens = {
+        'partial statement': [
+            include('preprocessor directives'),
+            (r'\d+e[+-]?\d+|(\d+\.\d*|\d*\.\d+)(e[+-]?\d+)?', Number.Float),
+            (r'0[xX][a-fA-F0-9]+', Number.Hex),
+            (r'\d+', Number.Integer),
+            (r'"', String, 'single-line string'),
+            (r'[+\-*/%^!=<>|&#~.]', Operator),
+            (r'[]()[]', Punctuation),
+            (r'\w+', Comment.Preproc)
+        ],
+
+        'statement': [
+            include('whitespace or macro call'),
+            newline + ('#pop',),
+            include('partial statement')
+        ],
+
+        'root': [
+            newline,
+            include('whitespace or macro call'),
+            (r'[{}]', Punctuation, 'statement'),
+            (r'[abefimq-tv-z]|[nN][pP]?', Keyword, 'statement')
+        ],
+
+        'single-line string': [
+            (r'"', String, '#pop'),
+            (r'[^\\"]+', String)
+        ]
+    }
+
+
+class CsoundOrchestraLexer(CsoundLexer):
+    """
+    For `Csound <http://csound.github.io>`_ orchestras.
+
+    .. versionadded:: 2.1
+    """
+
+    name = 'Csound Orchestra'
+    aliases = ['csound', 'csound-orc']
+    filenames = ['*.orc']
+
+    user_defined_opcodes = set()
+
+    def opcode_name_callback(lexer, match):
+        opcode = match.group(0)
+        lexer.user_defined_opcodes.add(opcode)
+        yield match.start(), Name.Function, opcode
+
+    def name_callback(lexer, match):
+        name = match.group(0)
+        if re.match('p\d+$', name) or name in OPCODES:
+            yield match.start(), Name.Builtin, name
+        elif name in lexer.user_defined_opcodes:
+            yield match.start(), Name.Function, name
+        else:
+            nameMatch = re.search(r'^(g?[aikSw])(\w+)', name)
+            if nameMatch:
+                yield nameMatch.start(1), Keyword.Type, nameMatch.group(1)
+                yield nameMatch.start(2), Name, nameMatch.group(2)
+            else:
+                yield match.start(), Name, name
+
+    tokens = {
+        'label': [
+            (r'\b(\w+)(:)', bygroups(Name.Label, Punctuation))
+        ],
+
+        'partial expression': [
+            include('preprocessor directives'),
+            (r'\b(0dbfs|k(r|smps)|nchnls(_i)?|sr)\b', Name.Variable.Global),
+            (r'\d+e[+-]?\d+|(\d+\.\d*|\d*\.\d+)(e[+-]?\d+)?', Number.Float),
+            (r'0[xX][a-fA-F0-9]+', Number.Hex),
+            (r'\d+', Number.Integer),
+            (r'"', String, 'single-line string'),
+            (r'\{\{', String, 'multi-line string'),
+            (r'[+\-*/%^!=&|<>#~¬]', Operator),
+            (r'[](),?:[]', Punctuation),
+            (words((
+                # Keywords
+                'do', 'else', 'elseif', 'endif', 'enduntil', 'fi', 'if', 'ithen', 'kthen',
+                'od', 'then', 'until', 'while',
+                # Opcodes that act as control structures
+                'return', 'timout'
+                ), prefix=r'\b', suffix=r'\b'), Keyword),
+            (words(('goto', 'igoto', 'kgoto', 'rigoto', 'tigoto'),
+                   prefix=r'\b', suffix=r'\b'), Keyword, 'goto label'),
+            (words(('cggoto', 'cigoto', 'cingoto', 'ckgoto', 'cngoto'),
+                   prefix=r'\b', suffix=r'\b'), Keyword,
+             ('goto label', 'goto expression')),
+            (words(('loop_ge', 'loop_gt', 'loop_le', 'loop_lt'),
+                   prefix=r'\b', suffix=r'\b'), Keyword,
+             ('goto label', 'goto expression', 'goto expression', 'goto expression')),
+            (r'\bscoreline(_i)?\b', Name.Builtin, 'scoreline opcode'),
+            (r'\bpyl?run[it]?\b', Name.Builtin, 'python opcode'),
+            (r'\blua_(exec|opdef)\b', Name.Builtin, 'lua opcode'),
+            (r'\b[a-zA-Z_]\w*\b', name_callback)
+        ],
+
+        'expression': [
+            include('whitespace or macro call'),
+            newline + ('#pop',),
+            include('partial expression')
+        ],
+
+        'root': [
+            newline,
+            include('whitespace or macro call'),
+            (r'\binstr\b', Keyword, ('instrument block', 'instrument name list')),
+            (r'\bopcode\b', Keyword, ('opcode block', 'opcode parameter list',
+                                      'opcode types', 'opcode types', 'opcode name')),
+            include('label'),
+            default('expression')
+        ],
+
+        'instrument name list': [
+            include('whitespace or macro call'),
+            (r'\d+|\+?[a-zA-Z_]\w*', Name.Function),
+            (r',', Punctuation),
+            newline + ('#pop',)
+        ],
+        'instrument block': [
+            newline,
+            include('whitespace or macro call'),
+            (r'\bendin\b', Keyword, '#pop'),
+            include('label'),
+            default('expression')
+        ],
+
+        'opcode name': [
+            include('whitespace or macro call'),
+            (r'[a-zA-Z_]\w*', opcode_name_callback, '#pop')
+        ],
+        'opcode types': [
+            include('whitespace or macro call'),
+            (r'0|[]afijkKoOpPStV[]+', Keyword.Type, '#pop'),
+            (r',', Punctuation)
+        ],
+        'opcode parameter list': [
+            include('whitespace or macro call'),
+            newline + ('#pop',)
+        ],
+        'opcode block': [
+            newline,
+            include('whitespace or macro call'),
+            (r'\bendop\b', Keyword, '#pop'),
+            include('label'),
+            default('expression')
+        ],
+
+        'goto label': [
+            include('whitespace or macro call'),
+            (r'\w+', Name.Label, '#pop'),
+            default('#pop')
+        ],
+        'goto expression': [
+            include('whitespace or macro call'),
+            (r',', Punctuation, '#pop'),
+            include('partial expression')
+        ],
+
+        'single-line string': [
+            include('macro call'),
+            (r'"', String, '#pop'),
+            # From https://github.com/csound/csound/blob/develop/Opcodes/fout.c#L1405
+            (r'%\d*(\.\d+)?[cdhilouxX]', String.Interpol),
+            (r'%[!%nNrRtT]|[~^]|\\([\\aAbBnNrRtT"]|[0-7]{1,3})', String.Escape),
+            (r'[^\\"~$%\^\n]+', String),
+            (r'[\\"~$%\^\n]', String)
+        ],
+        'multi-line string': [
+            (r'\}\}', String, '#pop'),
+            (r'[^}]+|\}(?!\})', String)
+        ],
+
+        'scoreline opcode': [
+            include('whitespace or macro call'),
+            (r'\{\{', String, 'scoreline'),
+            default('#pop')
+        ],
+        'scoreline': [
+            (r'\}\}', String, '#pop'),
+            (r'([^}]+)|\}(?!\})', using(CsoundScoreLexer))
+        ],
+
+        'python opcode': [
+            include('whitespace or macro call'),
+            (r'\{\{', String, 'python'),
+            default('#pop')
+        ],
+        'python': [
+            (r'\}\}', String, '#pop'),
+            (r'([^}]+)|\}(?!\})', using(PythonLexer))
+        ],
+
+        'lua opcode': [
+            include('whitespace or macro call'),
+            (r'"', String, 'single-line string'),
+            (r'\{\{', String, 'lua'),
+            (r',', Punctuation),
+            default('#pop')
+        ],
+        'lua': [
+            (r'\}\}', String, '#pop'),
+            (r'([^}]+)|\}(?!\})', using(LuaLexer))
+        ]
+    }
+
+
+class CsoundDocumentLexer(RegexLexer):
+    """
+    For `Csound <http://csound.github.io>`_ documents.
+
+    .. versionadded:: 2.1
+    """
+
+    name = 'Csound Document'
+    aliases = ['csound-document', 'csound-csd']
+    filenames = ['*.csd']
+
+    # These tokens are based on those in XmlLexer in pygments/lexers/html.py. Making
+    # CsoundDocumentLexer a subclass of XmlLexer rather than RegexLexer may seem like a
+    # better idea, since Csound Document files look like XML files. However, Csound
+    # Documents can contain Csound comments (preceded by //, for example) before and
+    # after the root element, unescaped bitwise AND & and less than < operators, etc. In
+    # other words, while Csound Document files look like XML files, they may not actually
+    # be XML files.
+    tokens = {
+        'root': [
+            newline,
+            (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+            (r'[^<&;/]+', Text),
+            (r'<\s*CsInstruments', Name.Tag, ('orchestra', 'tag')),
+            (r'<\s*CsScore', Name.Tag, ('score', 'tag')),
+            (r'<\s*[hH][tT][mM][lL]', Name.Tag, ('HTML', 'tag')),
+            (r'<\s*[\w:.-]+', Name.Tag, 'tag'),
+            (r'<\s*/\s*[\w:.-]+\s*>', Name.Tag)
+        ],
+        'orchestra': [
+            (r'<\s*/\s*CsInstruments\s*>', Name.Tag, '#pop'),
+            (r'(.|\n)+?(?=<\s*/\s*CsInstruments\s*>)', using(CsoundOrchestraLexer))
+        ],
+        'score': [
+            (r'<\s*/\s*CsScore\s*>', Name.Tag, '#pop'),
+            (r'(.|\n)+?(?=<\s*/\s*CsScore\s*>)', using(CsoundScoreLexer))
+        ],
+        'HTML': [
+            (r'<\s*/\s*[hH][tT][mM][lL]\s*>', Name.Tag, '#pop'),
+            (r'(.|\n)+?(?=<\s*/\s*[hH][tT][mM][lL]\s*>)', using(HtmlLexer))
+        ],
+        'tag': [
+            (r'\s+', Text),
+            (r'[\w.:-]+\s*=', Name.Attribute, 'attr'),
+            (r'/?\s*>', Name.Tag, '#pop')
+        ],
+        'attr': [
+            (r'\s+', Text),
+            (r'".*?"', String, '#pop'),
+            (r"'.*?'", String, '#pop'),
+            (r'[^\s>]+', String, '#pop')
+        ]
+    }

packages/wakatime/packages/pygments/lexers/css.py

@@ -0,0 +1,689 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.css
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for CSS and related stylesheet formats.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+import copy
+
+from pygments.lexer import ExtendedRegexLexer, RegexLexer, include, bygroups, \
+    default, words, inherit
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+from pygments.util import iteritems
+
+__all__ = ['CssLexer', 'SassLexer', 'ScssLexer', 'LessCssLexer']
+
+
+# List of vendor prefixes obtained from:
+# https://www.w3.org/TR/CSS21/syndata.html#vendor-keyword-history
+_vendor_prefixes = (
+    '-ms-', 'mso-', '-moz-', '-o-', '-xv-', '-atsc-', '-wap-', '-khtml-',
+    '-webkit-', 'prince-', '-ah-', '-hp-', '-ro-', '-rim-', '-tc-',
+)
+
+# List of CSS properties obtained from:
+# https://www.w3.org/Style/CSS/all-properties.en.html
+# Note: handle --* separately
+_css_properties = (
+    'align-content', 'align-items', 'align-self', 'alignment-baseline', 'all',
+    'animation', 'animation-delay', 'animation-direction',
+    'animation-duration', 'animation-fill-mode', 'animation-iteration-count',
+    'animation-name', 'animation-play-state', 'animation-timing-function',
+    'appearance', 'azimuth', 'backface-visibility', 'background',
+    'background-attachment', 'background-blend-mode', 'background-clip',
+    'background-color', 'background-image', 'background-origin',
+    'background-position', 'background-repeat', 'background-size',
+    'baseline-shift', 'bookmark-label', 'bookmark-level', 'bookmark-state',
+    'border', 'border-bottom', 'border-bottom-color',
+    'border-bottom-left-radius', 'border-bottom-right-radius',
+    'border-bottom-style', 'border-bottom-width', 'border-boundary',
+    'border-collapse', 'border-color', 'border-image', 'border-image-outset',
+    'border-image-repeat', 'border-image-slice', 'border-image-source',
+    'border-image-width', 'border-left', 'border-left-color',
+    'border-left-style', 'border-left-width', 'border-radius', 'border-right',
+    'border-right-color', 'border-right-style', 'border-right-width',
+    'border-spacing', 'border-style', 'border-top', 'border-top-color',
+    'border-top-left-radius', 'border-top-right-radius', 'border-top-style',
+    'border-top-width', 'border-width', 'bottom', 'box-decoration-break',
+    'box-shadow', 'box-sizing', 'box-snap', 'box-suppress', 'break-after',
+    'break-before', 'break-inside', 'caption-side', 'caret', 'caret-animation',
+    'caret-color', 'caret-shape', 'chains', 'clear', 'clip', 'clip-path',
+    'clip-rule', 'color', 'color-interpolation-filters', 'column-count',
+    'column-fill', 'column-gap', 'column-rule', 'column-rule-color',
+    'column-rule-style', 'column-rule-width', 'column-span', 'column-width',
+    'columns', 'content', 'counter-increment', 'counter-reset', 'counter-set',
+    'crop', 'cue', 'cue-after', 'cue-before', 'cursor', 'direction', 'display',
+    'dominant-baseline', 'elevation', 'empty-cells', 'filter', 'flex',
+    'flex-basis', 'flex-direction', 'flex-flow', 'flex-grow', 'flex-shrink',
+    'flex-wrap', 'float', 'float-defer', 'float-offset', 'float-reference',
+    'flood-color', 'flood-opacity', 'flow', 'flow-from', 'flow-into', 'font',
+    'font-family', 'font-feature-settings', 'font-kerning',
+    'font-language-override', 'font-size', 'font-size-adjust', 'font-stretch',
+    'font-style', 'font-synthesis', 'font-variant', 'font-variant-alternates',
+    'font-variant-caps', 'font-variant-east-asian', 'font-variant-ligatures',
+    'font-variant-numeric', 'font-variant-position', 'font-weight',
+    'footnote-display', 'footnote-policy', 'glyph-orientation-vertical',
+    'grid', 'grid-area', 'grid-auto-columns', 'grid-auto-flow',
+    'grid-auto-rows', 'grid-column', 'grid-column-end', 'grid-column-gap',
+    'grid-column-start', 'grid-gap', 'grid-row', 'grid-row-end',
+    'grid-row-gap', 'grid-row-start', 'grid-template', 'grid-template-areas',
+    'grid-template-columns', 'grid-template-rows', 'hanging-punctuation',
+    'height', 'hyphenate-character', 'hyphenate-limit-chars',
+    'hyphenate-limit-last', 'hyphenate-limit-lines', 'hyphenate-limit-zone',
+    'hyphens', 'image-orientation', 'image-resolution', 'initial-letter',
+    'initial-letter-align', 'initial-letter-wrap', 'isolation',
+    'justify-content', 'justify-items', 'justify-self', 'left',
+    'letter-spacing', 'lighting-color', 'line-break', 'line-grid',
+    'line-height', 'line-snap', 'list-style', 'list-style-image',
+    'list-style-position', 'list-style-type', 'margin', 'margin-bottom',
+    'margin-left', 'margin-right', 'margin-top', 'marker-side',
+    'marquee-direction', 'marquee-loop', 'marquee-speed', 'marquee-style',
+    'mask', 'mask-border', 'mask-border-mode', 'mask-border-outset',
+    'mask-border-repeat', 'mask-border-slice', 'mask-border-source',
+    'mask-border-width', 'mask-clip', 'mask-composite', 'mask-image',
+    'mask-mode', 'mask-origin', 'mask-position', 'mask-repeat', 'mask-size',
+    'mask-type', 'max-height', 'max-lines', 'max-width', 'min-height',
+    'min-width', 'mix-blend-mode', 'motion', 'motion-offset', 'motion-path',
+    'motion-rotation', 'move-to', 'nav-down', 'nav-left', 'nav-right',
+    'nav-up', 'object-fit', 'object-position', 'offset-after', 'offset-before',
+    'offset-end', 'offset-start', 'opacity', 'order', 'orphans', 'outline',
+    'outline-color', 'outline-offset', 'outline-style', 'outline-width',
+    'overflow', 'overflow-style', 'overflow-wrap', 'overflow-x', 'overflow-y',
+    'padding', 'padding-bottom', 'padding-left', 'padding-right', 'padding-top',
+    'page', 'page-break-after', 'page-break-before', 'page-break-inside',
+    'page-policy', 'pause', 'pause-after', 'pause-before', 'perspective',
+    'perspective-origin', 'pitch', 'pitch-range', 'play-during', 'polar-angle',
+    'polar-distance', 'position', 'presentation-level', 'quotes',
+    'region-fragment', 'resize', 'rest', 'rest-after', 'rest-before',
+    'richness', 'right', 'rotation', 'rotation-point', 'ruby-align',
+    'ruby-merge', 'ruby-position', 'running', 'scroll-snap-coordinate',
+    'scroll-snap-destination', 'scroll-snap-points-x', 'scroll-snap-points-y',
+    'scroll-snap-type', 'shape-image-threshold', 'shape-inside', 'shape-margin',
+    'shape-outside', 'size', 'speak', 'speak-as', 'speak-header',
+    'speak-numeral', 'speak-punctuation', 'speech-rate', 'stress', 'string-set',
+    'tab-size', 'table-layout', 'text-align', 'text-align-last',
+    'text-combine-upright', 'text-decoration', 'text-decoration-color',
+    'text-decoration-line', 'text-decoration-skip', 'text-decoration-style',
+    'text-emphasis', 'text-emphasis-color', 'text-emphasis-position',
+    'text-emphasis-style', 'text-indent', 'text-justify', 'text-orientation',
+    'text-overflow', 'text-shadow', 'text-space-collapse', 'text-space-trim',
+    'text-spacing', 'text-transform', 'text-underline-position', 'text-wrap',
+    'top', 'transform', 'transform-origin', 'transform-style', 'transition',
+    'transition-delay', 'transition-duration', 'transition-property',
+    'transition-timing-function', 'unicode-bidi', 'user-select',
+    'vertical-align', 'visibility', 'voice-balance', 'voice-duration',
+    'voice-family', 'voice-pitch', 'voice-range', 'voice-rate', 'voice-stress',
+    'voice-volume', 'volume', 'white-space', 'widows', 'width', 'will-change',
+    'word-break', 'word-spacing', 'word-wrap', 'wrap-after', 'wrap-before',
+    'wrap-flow', 'wrap-inside', 'wrap-through', 'writing-mode', 'z-index',
+)
+
+# List of keyword values obtained from: 
+# http://cssvalues.com/
+_keyword_values = (
+    'absolute', 'alias', 'all', 'all-petite-caps', 'all-scroll',
+    'all-small-caps', 'allow-end', 'alpha', 'alternate', 'alternate-reverse',
+    'always', 'armenian', 'auto', 'avoid', 'avoid-column', 'avoid-page',
+    'backwards', 'balance', 'baseline', 'below', 'blink', 'block', 'bold',
+    'bolder', 'border-box', 'both', 'bottom', 'box-decoration', 'break-word',
+    'capitalize', 'cell', 'center', 'circle', 'clip', 'clone', 'close-quote',
+    'col-resize', 'collapse', 'color', 'color-burn', 'color-dodge', 'column',
+    'column-reverse', 'compact', 'condensed', 'contain', 'container',
+    'content-box', 'context-menu', 'copy', 'cover', 'crisp-edges', 'crosshair',
+    'currentColor', 'cursive', 'darken', 'dashed', 'decimal',
+    'decimal-leading-zero', 'default', 'descendants', 'difference', 'digits',
+    'disc', 'distribute', 'dot', 'dotted', 'double', 'double-circle', 'e-resize',
+    'each-line', 'ease', 'ease-in', 'ease-in-out', 'ease-out', 'edges',
+    'ellipsis', 'end', 'ew-resize', 'exclusion', 'expanded', 'extra-condensed',
+    'extra-expanded', 'fantasy', 'fill', 'fill-box', 'filled', 'first', 'fixed',
+    'flat', 'flex', 'flex-end', 'flex-start', 'flip', 'force-end', 'forwards',
+    'from-image', 'full-width', 'geometricPrecision', 'georgian', 'groove',
+    'hanging', 'hard-light', 'help', 'hidden', 'hide', 'horizontal', 'hue',
+    'icon', 'infinite', 'inherit', 'initial', 'ink', 'inline', 'inline-block',
+    'inline-flex', 'inline-table', 'inset', 'inside', 'inter-word', 'invert',
+    'isolate', 'italic', 'justify', 'large', 'larger', 'last', 'left',
+    'lighten', 'lighter', 'line-through', 'linear', 'list-item', 'local',
+    'loose', 'lower-alpha', 'lower-greek', 'lower-latin', 'lower-roman',
+    'lowercase', 'ltr', 'luminance', 'luminosity', 'mandatory', 'manipulation',
+    'manual', 'margin-box', 'match-parent', 'medium', 'mixed', 'monospace',
+    'move', 'multiply', 'n-resize', 'ne-resize', 'nesw-resize',
+    'no-close-quote', 'no-drop', 'no-open-quote', 'no-repeat', 'none', 'normal',
+    'not-allowed', 'nowrap', 'ns-resize', 'nw-resize', 'nwse-resize', 'objects',
+    'oblique', 'off', 'on', 'open', 'open-quote', 'optimizeLegibility',
+    'optimizeSpeed', 'outset', 'outside', 'over', 'overlay', 'overline',
+    'padding-box', 'page', 'pan-down', 'pan-left', 'pan-right', 'pan-up',
+    'pan-x', 'pan-y', 'paused', 'petite-caps', 'pixelated', 'pointer',
+    'preserve-3d', 'progress', 'proximity', 'relative', 'repeat',
+    'repeat no-repeat', 'repeat-x', 'repeat-y', 'reverse', 'ridge', 'right',
+    'round', 'row', 'row-resize', 'row-reverse', 'rtl', 'ruby', 'ruby-base',
+    'ruby-base-container', 'ruby-text', 'ruby-text-container', 'run-in',
+    'running', 's-resize', 'sans-serif', 'saturation', 'scale-down', 'screen',
+    'scroll', 'se-resize', 'semi-condensed', 'semi-expanded', 'separate',
+    'serif', 'sesame', 'show', 'sideways', 'sideways-left', 'sideways-right',
+    'slice', 'small', 'small-caps', 'smaller', 'smooth', 'snap', 'soft-light',
+    'solid', 'space', 'space-around', 'space-between', 'spaces', 'square',
+    'start', 'static', 'step-end', 'step-start', 'sticky', 'stretch', 'strict',
+    'stroke-box', 'style', 'sw-resize', 'table', 'table-caption', 'table-cell',
+    'table-column', 'table-column-group', 'table-footer-group',
+    'table-header-group', 'table-row', 'table-row-group', 'text', 'thick',
+    'thin', 'titling-caps', 'to', 'top', 'triangle', 'ultra-condensed',
+    'ultra-expanded', 'under', 'underline', 'unicase', 'unset', 'upper-alpha',
+    'upper-latin', 'upper-roman', 'uppercase', 'upright', 'use-glyph-orientation',
+    'vertical', 'vertical-text', 'view-box', 'visible', 'w-resize', 'wait',
+    'wavy', 'weight', 'weight style', 'wrap', 'wrap-reverse', 'x-large',
+    'x-small', 'xx-large', 'xx-small', 'zoom-in', 'zoom-out',
+)
+
+# List of extended color keywords obtained from:
+# https://drafts.csswg.org/css-color/#named-colors
+_color_keywords = (
+    'aliceblue', 'antiquewhite', 'aqua', 'aquamarine', 'azure', 'beige',
+    'bisque', 'black', 'blanchedalmond', 'blue', 'blueviolet', 'brown',
+    'burlywood', 'cadetblue', 'chartreuse', 'chocolate', 'coral',
+    'cornflowerblue', 'cornsilk', 'crimson', 'cyan', 'darkblue', 'darkcyan',
+    'darkgoldenrod', 'darkgray', 'darkgreen', 'darkgrey', 'darkkhaki',
+    'darkmagenta', 'darkolivegreen', 'darkorange', 'darkorchid', 'darkred',
+    'darksalmon', 'darkseagreen', 'darkslateblue', 'darkslategray',
+    'darkslategrey', 'darkturquoise', 'darkviolet', 'deeppink', 'deepskyblue',
+    'dimgray', 'dimgrey', 'dodgerblue', 'firebrick', 'floralwhite',
+    'forestgreen', 'fuchsia', 'gainsboro', 'ghostwhite', 'gold', 'goldenrod',
+    'gray', 'green', 'greenyellow', 'grey', 'honeydew', 'hotpink', 'indianred',
+    'indigo', 'ivory', 'khaki', 'lavender', 'lavenderblush', 'lawngreen',
+    'lemonchiffon', 'lightblue', 'lightcoral', 'lightcyan',
+    'lightgoldenrodyellow', 'lightgray', 'lightgreen', 'lightgrey',
+    'lightpink', 'lightsalmon', 'lightseagreen', 'lightskyblue',
+    'lightslategray', 'lightslategrey', 'lightsteelblue', 'lightyellow',
+    'lime', 'limegreen', 'linen', 'magenta', 'maroon', 'mediumaquamarine',
+    'mediumblue', 'mediumorchid', 'mediumpurple', 'mediumseagreen',
+    'mediumslateblue', 'mediumspringgreen', 'mediumturquoise',
+    'mediumvioletred', 'midnightblue', 'mintcream', 'mistyrose', 'moccasin',
+    'navajowhite', 'navy', 'oldlace', 'olive', 'olivedrab', 'orange',
+    'orangered', 'orchid', 'palegoldenrod', 'palegreen', 'paleturquoise',
+    'palevioletred', 'papayawhip', 'peachpuff', 'peru', 'pink', 'plum',
+    'powderblue', 'purple', 'rebeccapurple', 'red', 'rosybrown', 'royalblue',
+    'saddlebrown', 'salmon', 'sandybrown', 'seagreen', 'seashell', 'sienna',
+    'silver', 'skyblue', 'slateblue', 'slategray', 'slategrey', 'snow',
+    'springgreen', 'steelblue', 'tan', 'teal', 'thistle', 'tomato', 'turquoise',
+    'violet', 'wheat', 'white', 'whitesmoke', 'yellow', 'yellowgreen',
+) + ('transparent',)
+
+# List of other keyword values from other sources:
+_other_keyword_values = (
+    'above', 'aural', 'behind', 'bidi-override', 'center-left', 'center-right',
+    'cjk-ideographic', 'continuous', 'crop', 'cross', 'embed', 'far-left',
+    'far-right', 'fast', 'faster', 'hebrew', 'high', 'higher', 'hiragana',
+    'hiragana-iroha', 'katakana', 'katakana-iroha', 'landscape', 'left-side',
+    'leftwards', 'level', 'loud', 'low', 'lower', 'message-box', 'middle',
+    'mix', 'narrower', 'once', 'portrait', 'right-side', 'rightwards', 'silent',
+    'slow', 'slower', 'small-caption', 'soft', 'spell-out', 'status-bar',
+    'super', 'text-bottom', 'text-top', 'wider', 'x-fast', 'x-high', 'x-loud',
+    'x-low', 'x-soft', 'yes', 'pre', 'pre-wrap', 'pre-line',
+)
+
+# List of functional notation and function keyword values:
+_functional_notation_keyword_values = (
+    'attr', 'blackness', 'blend', 'blenda', 'blur', 'brightness', 'calc',
+    'circle', 'color-mod', 'contrast', 'counter', 'cubic-bezier', 'device-cmyk',
+    'drop-shadow', 'ellipse', 'gray', 'grayscale', 'hsl', 'hsla', 'hue',
+    'hue-rotate', 'hwb', 'image', 'inset', 'invert', 'lightness',
+    'linear-gradient', 'matrix', 'matrix3d', 'opacity', 'perspective',
+    'polygon', 'radial-gradient', 'rect', 'repeating-linear-gradient',
+    'repeating-radial-gradient', 'rgb', 'rgba', 'rotate', 'rotate3d', 'rotateX',
+    'rotateY', 'rotateZ', 'saturate', 'saturation', 'scale', 'scale3d',
+    'scaleX', 'scaleY', 'scaleZ', 'sepia', 'shade', 'skewX', 'skewY', 'steps',
+    'tint', 'toggle', 'translate', 'translate3d', 'translateX', 'translateY',
+    'translateZ', 'whiteness',
+)
+# Note! Handle url(...) separately.
+
+# List of units obtained from:
+# https://www.w3.org/TR/css3-values/
+_angle_units = (
+    'deg', 'grad', 'rad', 'turn',
+)
+_frequency_units = (
+    'Hz', 'kHz',
+)
+_length_units = (
+    'em', 'ex', 'ch', 'rem',
+    'vh', 'vw', 'vmin', 'vmax',
+    'px', 'mm', 'cm', 'in', 'pt', 'pc', 'q',
+)
+_resolution_units = (
+    'dpi', 'dpcm', 'dppx',
+)
+_time_units = (
+    's', 'ms',
+)
+_all_units = _angle_units + _frequency_units + _length_units + \
+             _resolution_units + _time_units
+
+
+class CssLexer(RegexLexer):
+    """
+    For CSS (Cascading Style Sheets).
+    """
+
+    name = 'CSS'
+    aliases = ['css']
+    filenames = ['*.css']
+    mimetypes = ['text/css']
+
+    tokens = {
+        'root': [
+            include('basics'),
+        ],
+        'basics': [
+            (r'\s+', Text),
+            (r'/\*(?:.|\n)*?\*/', Comment),
+            (r'\{', Punctuation, 'content'),
+            (r'(\:{1,2})([\w-]+)', bygroups(Punctuation, Name.Decorator)),
+            (r'(\.)([\w-]+)', bygroups(Punctuation, Name.Class)),
+            (r'(\#)([\w-]+)', bygroups(Punctuation, Name.Namespace)),
+            (r'(@)([\w-]+)', bygroups(Punctuation, Keyword), 'atrule'),
+            (r'[\w-]+', Name.Tag),
+            (r'[~^*!%&$\[\]()<>|+=@:;,./?-]', Operator),
+            (r'"(\\\\|\\"|[^"])*"', String.Double),
+            (r"'(\\\\|\\'|[^'])*'", String.Single)
+        ],
+        'atrule': [
+            (r'\{', Punctuation, 'atcontent'),
+            (r';', Punctuation, '#pop'),
+            include('basics'),
+        ],
+        'atcontent': [
+            include('basics'),
+            (r'\}', Punctuation, '#pop:2'),
+        ],
+        'content': [
+            (r'\s+', Text),
+            (r'\}', Punctuation, '#pop'),
+            (r';', Punctuation),
+            (r'^@.*?$', Comment.Preproc),
+
+            (words(_vendor_prefixes,), Keyword.Pseudo),
+            (r'('+r'|'.join(_css_properties)+r')(\s*)(\:)',
+             bygroups(Keyword, Text, Punctuation), 'value-start'),
+            (r'([a-zA-Z_][\w-]*)(\s*)(\:)', bygroups(Name, Text, Punctuation),
+             'value-start'),
+
+            (r'/\*(?:.|\n)*?\*/', Comment),
+        ],
+        'value-start': [
+            (r'\s+', Text),
+            (words(_vendor_prefixes,), Name.Builtin.Pseudo),
+            include('urls'),
+            (r'('+r'|'.join(_functional_notation_keyword_values)+r')(\()',
+             bygroups(Name.Builtin, Punctuation), 'function-start'),
+            (r'([a-zA-Z_][\w-]+)(\()', bygroups(Name.Function, Punctuation), 'function-start'),
+            (words(_keyword_values, suffix=r'\b'), Keyword.Constant),
+            (words(_other_keyword_values, suffix=r'\b'), Keyword.Constant),
+            (words(_color_keywords, suffix=r'\b'), Keyword.Constant),
+            (words(_css_properties, suffix=r'\b'), Keyword), # for transition-property etc.
+            (r'\!important', Comment.Preproc),
+            (r'/\*(?:.|\n)*?\*/', Comment),
+
+            include('numeric-values'),
+            
+            (r'[~^*!%&<>|+=@:./?-]+', Operator),
+            (r'[\[\](),]+', Punctuation),
+            (r'"(\\\\|\\"|[^"])*"', String.Double),
+            (r"'(\\\\|\\'|[^'])*'", String.Single),
+            (r'[a-zA-Z_][\w-]*', Name),
+            (r';', Punctuation, '#pop'),
+            (r'\}', Punctuation, '#pop:2'),
+        ],
+        'function-start': [
+            (r'\s+', Text),
+            include('urls'),
+            (words(_vendor_prefixes,), Keyword.Pseudo),
+            (words(_keyword_values, suffix=r'\b'), Keyword.Constant),
+            (words(_other_keyword_values, suffix=r'\b'), Keyword.Constant),
+            (words(_color_keywords, suffix=r'\b'), Keyword.Constant),
+
+            # function-start may be entered recursively
+            (r'(' + r'|'.join(_functional_notation_keyword_values) + r')(\()',
+             bygroups(Name.Builtin, Punctuation), 'function-start'),
+            (r'([a-zA-Z_][\w-]+)(\()', bygroups(Name.Function, Punctuation), 'function-start'),
+
+            (r'/\*(?:.|\n)*?\*/', Comment),
+            include('numeric-values'),
+            (r'[*+/-]', Operator),
+            (r'[,]', Punctuation),
+            (r'"(\\\\|\\"|[^"])*"', String.Double),
+            (r"'(\\\\|\\'|[^'])*'", String.Single),
+            (r'[a-zA-Z_-]\w*', Name),
+            (r'\)', Punctuation, '#pop'),
+        ],
+        'urls': [
+            (r'(url)(\()(".*?")(\))', bygroups(Name.Builtin, Punctuation,
+                                               String.Double, Punctuation)),
+            (r"(url)(\()('.*?')(\))", bygroups(Name.Builtin, Punctuation,
+                                               String.Single, Punctuation)),
+            (r'(url)(\()(.*?)(\))', bygroups(Name.Builtin, Punctuation,
+                                             String.Other, Punctuation)),
+        ],
+        'numeric-values': [
+            (r'\#[a-zA-Z0-9]{1,6}', Number.Hex),
+            (r'[+\-]?[0-9]*[.][0-9]+', Number.Float, 'numeric-end'),
+            (r'[+\-]?[0-9]+', Number.Integer, 'numeric-end'),            
+        ],        
+        'numeric-end': [
+            (words(_all_units, suffix=r'\b'), Keyword.Type),
+            (r'%', Keyword.Type),
+            default('#pop'),
+        ],
+    }
+
+
+common_sass_tokens = {
+    'value': [
+        (r'[ \t]+', Text),
+        (r'[!$][\w-]+', Name.Variable),
+        (r'url\(', String.Other, 'string-url'),
+        (r'[a-z_-][\w-]*(?=\()', Name.Function),
+        (words(_css_properties + (
+            'above', 'absolute', 'always', 'armenian', 'aural', 'auto', 'avoid', 'baseline',
+            'behind', 'below', 'bidi-override', 'blink', 'block', 'bold', 'bolder', 'both',
+            'capitalize', 'center-left', 'center-right', 'center', 'circle',
+            'cjk-ideographic', 'close-quote', 'collapse', 'condensed', 'continuous',
+            'crop', 'crosshair', 'cross', 'cursive', 'dashed', 'decimal-leading-zero',
+            'decimal', 'default', 'digits', 'disc', 'dotted', 'double', 'e-resize', 'embed',
+            'extra-condensed', 'extra-expanded', 'expanded', 'fantasy', 'far-left',
+            'far-right', 'faster', 'fast', 'fixed', 'georgian', 'groove', 'hebrew', 'help',
+            'hidden', 'hide', 'higher', 'high', 'hiragana-iroha', 'hiragana', 'icon',
+            'inherit', 'inline-table', 'inline', 'inset', 'inside', 'invert', 'italic',
+            'justify', 'katakana-iroha', 'katakana', 'landscape', 'larger', 'large',
+            'left-side', 'leftwards', 'level', 'lighter', 'line-through', 'list-item',
+            'loud', 'lower-alpha', 'lower-greek', 'lower-roman', 'lowercase', 'ltr',
+            'lower', 'low', 'medium', 'message-box', 'middle', 'mix', 'monospace',
+            'n-resize', 'narrower', 'ne-resize', 'no-close-quote', 'no-open-quote',
+            'no-repeat', 'none', 'normal', 'nowrap', 'nw-resize', 'oblique', 'once',
+            'open-quote', 'outset', 'outside', 'overline', 'pointer', 'portrait', 'px',
+            'relative', 'repeat-x', 'repeat-y', 'repeat', 'rgb', 'ridge', 'right-side',
+            'rightwards', 's-resize', 'sans-serif', 'scroll', 'se-resize',
+            'semi-condensed', 'semi-expanded', 'separate', 'serif', 'show', 'silent',
+            'slow', 'slower', 'small-caps', 'small-caption', 'smaller', 'soft', 'solid',
+            'spell-out', 'square', 'static', 'status-bar', 'super', 'sw-resize',
+            'table-caption', 'table-cell', 'table-column', 'table-column-group',
+            'table-footer-group', 'table-header-group', 'table-row',
+            'table-row-group', 'text', 'text-bottom', 'text-top', 'thick', 'thin',
+            'transparent', 'ultra-condensed', 'ultra-expanded', 'underline',
+            'upper-alpha', 'upper-latin', 'upper-roman', 'uppercase', 'url',
+            'visible', 'w-resize', 'wait', 'wider', 'x-fast', 'x-high', 'x-large', 'x-loud',
+            'x-low', 'x-small', 'x-soft', 'xx-large', 'xx-small', 'yes'), suffix=r'\b'),
+         Name.Constant),
+        (words(_color_keywords, suffix=r'\b'), Name.Entity),
+        (words((
+            'black', 'silver', 'gray', 'white', 'maroon', 'red', 'purple', 'fuchsia', 'green',
+            'lime', 'olive', 'yellow', 'navy', 'blue', 'teal', 'aqua'), suffix=r'\b'),
+         Name.Builtin),
+        (r'\!(important|default)', Name.Exception),
+        (r'(true|false)', Name.Pseudo),
+        (r'(and|or|not)', Operator.Word),
+        (r'/\*', Comment.Multiline, 'inline-comment'),
+        (r'//[^\n]*', Comment.Single),
+        (r'\#[a-z0-9]{1,6}', Number.Hex),
+        (r'(-?\d+)(\%|[a-z]+)?', bygroups(Number.Integer, Keyword.Type)),
+        (r'(-?\d*\.\d+)(\%|[a-z]+)?', bygroups(Number.Float, Keyword.Type)),
+        (r'#\{', String.Interpol, 'interpolation'),
+        (r'[~^*!&%<>|+=@:,./?-]+', Operator),
+        (r'[\[\]()]+', Punctuation),
+        (r'"', String.Double, 'string-double'),
+        (r"'", String.Single, 'string-single'),
+        (r'[a-z_-][\w-]*', Name),
+    ],
+
+    'interpolation': [
+        (r'\}', String.Interpol, '#pop'),
+        include('value'),
+    ],
+
+    'selector': [
+        (r'[ \t]+', Text),
+        (r'\:', Name.Decorator, 'pseudo-class'),
+        (r'\.', Name.Class, 'class'),
+        (r'\#', Name.Namespace, 'id'),
+        (r'[\w-]+', Name.Tag),
+        (r'#\{', String.Interpol, 'interpolation'),
+        (r'&', Keyword),
+        (r'[~^*!&\[\]()<>|+=@:;,./?-]', Operator),
+        (r'"', String.Double, 'string-double'),
+        (r"'", String.Single, 'string-single'),
+    ],
+
+    'string-double': [
+        (r'(\\.|#(?=[^\n{])|[^\n"#])+', String.Double),
+        (r'#\{', String.Interpol, 'interpolation'),
+        (r'"', String.Double, '#pop'),
+    ],
+
+    'string-single': [
+        (r"(\\.|#(?=[^\n{])|[^\n'#])+", String.Double),
+        (r'#\{', String.Interpol, 'interpolation'),
+        (r"'", String.Double, '#pop'),
+    ],
+
+    'string-url': [
+        (r'(\\#|#(?=[^\n{])|[^\n#)])+', String.Other),
+        (r'#\{', String.Interpol, 'interpolation'),
+        (r'\)', String.Other, '#pop'),
+    ],
+
+    'pseudo-class': [
+        (r'[\w-]+', Name.Decorator),
+        (r'#\{', String.Interpol, 'interpolation'),
+        default('#pop'),
+    ],
+
+    'class': [
+        (r'[\w-]+', Name.Class),
+        (r'#\{', String.Interpol, 'interpolation'),
+        default('#pop'),
+    ],
+
+    'id': [
+        (r'[\w-]+', Name.Namespace),
+        (r'#\{', String.Interpol, 'interpolation'),
+        default('#pop'),
+    ],
+
+    'for': [
+        (r'(from|to|through)', Operator.Word),
+        include('value'),
+    ],
+}
+
+
+def _indentation(lexer, match, ctx):
+    indentation = match.group(0)
+    yield match.start(), Text, indentation
+    ctx.last_indentation = indentation
+    ctx.pos = match.end()
+
+    if hasattr(ctx, 'block_state') and ctx.block_state and \
+            indentation.startswith(ctx.block_indentation) and \
+            indentation != ctx.block_indentation:
+        ctx.stack.append(ctx.block_state)
+    else:
+        ctx.block_state = None
+        ctx.block_indentation = None
+        ctx.stack.append('content')
+
+
+def _starts_block(token, state):
+    def callback(lexer, match, ctx):
+        yield match.start(), token, match.group(0)
+
+        if hasattr(ctx, 'last_indentation'):
+            ctx.block_indentation = ctx.last_indentation
+        else:
+            ctx.block_indentation = ''
+
+        ctx.block_state = state
+        ctx.pos = match.end()
+
+    return callback
+
+
+class SassLexer(ExtendedRegexLexer):
+    """
+    For Sass stylesheets.
+
+    .. versionadded:: 1.3
+    """
+
+    name = 'Sass'
+    aliases = ['sass']
+    filenames = ['*.sass']
+    mimetypes = ['text/x-sass']
+
+    flags = re.IGNORECASE | re.MULTILINE
+
+    tokens = {
+        'root': [
+            (r'[ \t]*\n', Text),
+            (r'[ \t]*', _indentation),
+        ],
+
+        'content': [
+            (r'//[^\n]*', _starts_block(Comment.Single, 'single-comment'),
+             'root'),
+            (r'/\*[^\n]*', _starts_block(Comment.Multiline, 'multi-comment'),
+             'root'),
+            (r'@import', Keyword, 'import'),
+            (r'@for', Keyword, 'for'),
+            (r'@(debug|warn|if|while)', Keyword, 'value'),
+            (r'(@mixin)( [\w-]+)', bygroups(Keyword, Name.Function), 'value'),
+            (r'(@include)( [\w-]+)', bygroups(Keyword, Name.Decorator), 'value'),
+            (r'@extend', Keyword, 'selector'),
+            (r'@[\w-]+', Keyword, 'selector'),
+            (r'=[\w-]+', Name.Function, 'value'),
+            (r'\+[\w-]+', Name.Decorator, 'value'),
+            (r'([!$][\w-]\w*)([ \t]*(?:(?:\|\|)?=|:))',
+             bygroups(Name.Variable, Operator), 'value'),
+            (r':', Name.Attribute, 'old-style-attr'),
+            (r'(?=.+?[=:]([^a-z]|$))', Name.Attribute, 'new-style-attr'),
+            default('selector'),
+        ],
+
+        'single-comment': [
+            (r'.+', Comment.Single),
+            (r'\n', Text, 'root'),
+        ],
+
+        'multi-comment': [
+            (r'.+', Comment.Multiline),
+            (r'\n', Text, 'root'),
+        ],
+
+        'import': [
+            (r'[ \t]+', Text),
+            (r'\S+', String),
+            (r'\n', Text, 'root'),
+        ],
+
+        'old-style-attr': [
+            (r'[^\s:="\[]+', Name.Attribute),
+            (r'#\{', String.Interpol, 'interpolation'),
+            (r'[ \t]*=', Operator, 'value'),
+            default('value'),
+        ],
+
+        'new-style-attr': [
+            (r'[^\s:="\[]+', Name.Attribute),
+            (r'#\{', String.Interpol, 'interpolation'),
+            (r'[ \t]*[=:]', Operator, 'value'),
+        ],
+
+        'inline-comment': [
+            (r"(\\#|#(?=[^\n{])|\*(?=[^\n/])|[^\n#*])+", Comment.Multiline),
+            (r'#\{', String.Interpol, 'interpolation'),
+            (r"\*/", Comment, '#pop'),
+        ],
+    }
+    for group, common in iteritems(common_sass_tokens):
+        tokens[group] = copy.copy(common)
+    tokens['value'].append((r'\n', Text, 'root'))
+    tokens['selector'].append((r'\n', Text, 'root'))
+
+
+class ScssLexer(RegexLexer):
+    """
+    For SCSS stylesheets.
+    """
+
+    name = 'SCSS'
+    aliases = ['scss']
+    filenames = ['*.scss']
+    mimetypes = ['text/x-scss']
+
+    flags = re.IGNORECASE | re.DOTALL
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+            (r'//.*?\n', Comment.Single),
+            (r'/\*.*?\*/', Comment.Multiline),
+            (r'@import', Keyword, 'value'),
+            (r'@for', Keyword, 'for'),
+            (r'@(debug|warn|if|while)', Keyword, 'value'),
+            (r'(@mixin)( [\w-]+)', bygroups(Keyword, Name.Function), 'value'),
+            (r'(@include)( [\w-]+)', bygroups(Keyword, Name.Decorator), 'value'),
+            (r'@extend', Keyword, 'selector'),
+            (r'(@media)(\s+)', bygroups(Keyword, Text), 'value'),
+            (r'@[\w-]+', Keyword, 'selector'),
+            (r'(\$[\w-]*\w)([ \t]*:)', bygroups(Name.Variable, Operator), 'value'),
+            # TODO: broken, and prone to infinite loops.
+            # (r'(?=[^;{}][;}])', Name.Attribute, 'attr'),
+            # (r'(?=[^;{}:]+:[^a-z])', Name.Attribute, 'attr'),
+            default('selector'),
+        ],
+
+        'attr': [
+            (r'[^\s:="\[]+', Name.Attribute),
+            (r'#\{', String.Interpol, 'interpolation'),
+            (r'[ \t]*:', Operator, 'value'),
+            default('#pop'),
+        ],
+
+        'inline-comment': [
+            (r"(\\#|#(?=[^{])|\*(?=[^/])|[^#*])+", Comment.Multiline),
+            (r'#\{', String.Interpol, 'interpolation'),
+            (r"\*/", Comment, '#pop'),
+        ],
+    }
+    for group, common in iteritems(common_sass_tokens):
+        tokens[group] = copy.copy(common)
+    tokens['value'].extend([(r'\n', Text), (r'[;{}]', Punctuation, '#pop')])
+    tokens['selector'].extend([(r'\n', Text), (r'[;{}]', Punctuation, '#pop')])
+
+
+class LessCssLexer(CssLexer):
+    """
+    For `LESS <http://lesscss.org/>`_ styleshets.
+
+    .. versionadded:: 2.1
+    """
+
+    name = 'LessCss'
+    aliases = ['less']
+    filenames = ['*.less']
+    mimetypes = ['text/x-less-css']
+
+    tokens = {
+        'root': [
+            (r'@\w+', Name.Variable),
+            inherit,
+        ],
+        'content': [
+            (r'\{', Punctuation, '#push'),
+            inherit,
+        ],
+    }

packages/wakatime/packages/pygments/lexers/d.py

@@ -5,7 +5,7 @@
 
     Lexers for D languages.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/dalvik.py

@@ -5,7 +5,7 @@
 
     Pygments lexers for Dalvik VM-related languages.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/data.py

@@ -5,7 +5,7 @@
 
     Lexers for data file format.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -14,9 +14,9 @@ import re
 from pygments.lexer import RegexLexer, ExtendedRegexLexer, LexerContext, \
     include, bygroups, inherit
 from pygments.token import Text, Comment, Keyword, Name, String, Number, \
-    Punctuation, Literal
+    Punctuation, Literal, Error
 
-__all__ = ['YamlLexer', 'JsonLexer', 'JsonLdLexer']
+__all__ = ['YamlLexer', 'JsonLexer', 'JsonBareObjectLexer', 'JsonLdLexer']
 
 
 class YamlLexerContext(LexerContext):
@@ -247,10 +247,10 @@ class YamlLexer(ExtendedRegexLexer):
         # tags, anchors, aliases
         'descriptors': [
             # a full-form tag
-            (r'!<[\w;/?:@&=+$,.!~*\'()\[\]%-]+>', Keyword.Type),
+            (r'!<[\w#;/?:@&=+$,.!~*\'()\[\]%-]+>', Keyword.Type),
             # a tag in the form '!', '!suffix' or '!handle!suffix'
-            (r'!(?:[\w-]+)?'
-             r'(?:![\w;/?:@&=+$,.!~*\'()\[\]%-]+)?', Keyword.Type),
+            (r'!(?:[\w-]+!)?'
+             r'[\w#;/?:@&=+$,.!~*\'()\[\]%-]+', Keyword.Type),
             # an anchor
             (r'&[\w-]+', Name.Label),
             # an alias
@@ -476,7 +476,7 @@ class JsonLexer(RegexLexer):
             # comma terminates the attribute but expects more
             (r',', Punctuation, '#pop'),
             # a closing bracket terminates the entire object, so pop twice
-            (r'\}', Punctuation, ('#pop', '#pop')),
+            (r'\}', Punctuation, '#pop:2'),
         ],
 
         # a json object - { attr, attr, ... }
@@ -508,6 +508,31 @@ class JsonLexer(RegexLexer):
         ],
     }
 
+
+class JsonBareObjectLexer(JsonLexer):
+    """
+    For JSON data structures (with missing object curly braces).
+
+    .. versionadded:: 2.2
+    """
+
+    name = 'JSONBareObject'
+    aliases = ['json-object']
+    filenames = []
+    mimetypes = ['application/json-object']
+
+    tokens = {
+        'root': [
+            (r'\}', Error),
+            include('objectvalue'),
+        ],
+        'objectattribute': [
+            (r'\}', Error),
+            inherit,
+        ],
+    }
+
+
 class JsonLdLexer(JsonLexer):
     """
     For `JSON-LD <http://json-ld.org/>`_ linked data.

packages/wakatime/packages/pygments/lexers/diff.py

@@ -5,15 +5,17 @@
 
     Lexers for diff/patch formats.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+import re
+
 from pygments.lexer import RegexLexer, include, bygroups
 from pygments.token import Text, Comment, Operator, Keyword, Name, Generic, \
     Literal
 
-__all__ = ['DiffLexer', 'DarcsPatchLexer']
+__all__ = ['DiffLexer', 'DarcsPatchLexer', 'WDiffLexer']
 
 
 class DiffLexer(RegexLexer):
@@ -104,3 +106,60 @@ class DarcsPatchLexer(RegexLexer):
             (r'[^\n\[]+', Generic.Deleted),
         ],
     }
+
+
+class WDiffLexer(RegexLexer):
+    """
+    A `wdiff <https://www.gnu.org/software/wdiff/>`_ lexer.
+
+    Note that:
+
+    * only to normal output (without option like -l).
+    * if target files of wdiff contain "[-", "-]", "{+", "+}",
+      especially they are unbalanced, this lexer will get confusing.
+
+    .. versionadded:: 2.2
+    """
+
+    name = 'WDiff'
+    aliases = ['wdiff']
+    filenames = ['*.wdiff']
+    mimetypes = []
+
+    flags = re.MULTILINE | re.DOTALL
+
+    # We can only assume "[-" after "[-" before "-]" is `nested`,
+    # for instance wdiff to wdiff outputs. We have no way to
+    # distinct these marker is of wdiff output from original text.
+
+    ins_op = r"\{\+"
+    ins_cl = r"\+\}"
+    del_op = r"\[\-"
+    del_cl = r"\-\]"
+    normal = r'[^{}[\]+-]+'  # for performance
+    tokens = {
+        'root': [
+            (ins_op, Generic.Inserted, 'inserted'),
+            (del_op, Generic.Deleted, 'deleted'),
+            (normal, Text),
+            (r'.', Text),
+        ],
+        'inserted': [
+            (ins_op, Generic.Inserted, '#push'),
+            (del_op, Generic.Inserted, '#push'),
+            (del_cl, Generic.Inserted, '#pop'),
+
+            (ins_cl, Generic.Inserted, '#pop'),
+            (normal, Generic.Inserted),
+            (r'.', Generic.Inserted),
+        ],
+        'deleted': [
+            (del_op, Generic.Deleted, '#push'),
+            (ins_op, Generic.Deleted, '#push'),
+            (ins_cl, Generic.Deleted, '#pop'),
+
+            (del_cl, Generic.Deleted, '#pop'),
+            (normal, Generic.Deleted),
+            (r'.', Generic.Deleted),
+        ],
+    }

packages/wakatime/packages/pygments/lexers/dotnet.py

@@ -5,13 +5,13 @@
 
     Lexers for .net languages.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 import re
 
 from pygments.lexer import RegexLexer, DelegatingLexer, bygroups, include, \
-    using, this, default
+    using, this, default, words
 from pygments.token import Punctuation, \
     Text, Comment, Operator, Keyword, Name, String, Number, Literal, Other
 from pygments.util import get_choice_opt, iteritems
@@ -97,17 +97,17 @@ class CSharpLexer(RegexLexer):
                  Comment.Preproc),
                 (r'\b(extern)(\s+)(alias)\b', bygroups(Keyword, Text,
                  Keyword)),
-                (r'(abstract|as|async|await|base|break|case|catch|'
+                (r'(abstract|as|async|await|base|break|by|case|catch|'
                  r'checked|const|continue|default|delegate|'
                  r'do|else|enum|event|explicit|extern|false|finally|'
                  r'fixed|for|foreach|goto|if|implicit|in|interface|'
-                 r'internal|is|lock|new|null|operator|'
+                 r'internal|is|let|lock|new|null|on|operator|'
                  r'out|override|params|private|protected|public|readonly|'
                  r'ref|return|sealed|sizeof|stackalloc|static|'
                  r'switch|this|throw|true|try|typeof|'
                  r'unchecked|unsafe|virtual|void|while|'
                  r'get|set|new|partial|yield|add|remove|value|alias|ascending|'
-                 r'descending|from|group|into|orderby|select|where|'
+                 r'descending|from|group|into|orderby|select|thenby|where|'
                  r'join|equals)\b', Keyword),
                 (r'(global)(::)', bygroups(Keyword, Punctuation)),
                 (r'(bool|byte|char|decimal|double|dynamic|float|int|long|object|'
@@ -375,8 +375,8 @@ class VbNetLexer(RegexLexer):
     filenames = ['*.vb', '*.bas']
     mimetypes = ['text/x-vbnet', 'text/x-vba']  # (?)
 
-    uni_name = '[_' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl') + ']' + \
-               '[' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Nl', 'Nd', 'Pc',
+    uni_name = '[_' + uni.combine('Ll', 'Lt', 'Lm', 'Nl') + ']' + \
+               '[' + uni.combine('Ll', 'Lt', 'Lm', 'Nl', 'Nd', 'Pc',
                                  'Cf', 'Mn', 'Mc') + ']*'
 
     flags = re.MULTILINE | re.IGNORECASE
@@ -394,25 +394,26 @@ class VbNetLexer(RegexLexer):
             (r'[(){}!#,.:]', Punctuation),
             (r'Option\s+(Strict|Explicit|Compare)\s+'
              r'(On|Off|Binary|Text)', Keyword.Declaration),
-            (r'(?<!\.)(AddHandler|Alias|'
-             r'ByRef|ByVal|Call|Case|Catch|CBool|CByte|CChar|CDate|'
-             r'CDec|CDbl|CInt|CLng|CObj|Continue|CSByte|CShort|'
-             r'CSng|CStr|CType|CUInt|CULng|CUShort|Declare|'
-             r'Default|Delegate|DirectCast|Do|Each|Else|ElseIf|'
-             r'EndIf|Erase|Error|Event|Exit|False|Finally|For|'
-             r'Friend|Get|Global|GoSub|GoTo|Handles|If|'
-             r'Implements|Inherits|Interface|'
-             r'Let|Lib|Loop|Me|MustInherit|'
-             r'MustOverride|MyBase|MyClass|Narrowing|New|Next|'
-             r'Not|Nothing|NotInheritable|NotOverridable|Of|On|'
-             r'Operator|Option|Optional|Overloads|Overridable|'
-             r'Overrides|ParamArray|Partial|Private|Protected|'
-             r'Public|RaiseEvent|ReadOnly|ReDim|RemoveHandler|Resume|'
-             r'Return|Select|Set|Shadows|Shared|Single|'
-             r'Static|Step|Stop|SyncLock|Then|'
-             r'Throw|To|True|Try|TryCast|Wend|'
-             r'Using|When|While|Widening|With|WithEvents|'
-             r'WriteOnly)\b', Keyword),
+            (words((
+                'AddHandler', 'Alias', 'ByRef', 'ByVal', 'Call', 'Case',
+                'Catch', 'CBool', 'CByte', 'CChar', 'CDate', 'CDec', 'CDbl',
+                'CInt', 'CLng', 'CObj', 'Continue', 'CSByte', 'CShort', 'CSng',
+                'CStr', 'CType', 'CUInt', 'CULng', 'CUShort', 'Declare',
+                'Default', 'Delegate', 'DirectCast', 'Do', 'Each', 'Else',
+                'ElseIf', 'EndIf', 'Erase', 'Error', 'Event', 'Exit', 'False',
+                'Finally', 'For', 'Friend', 'Get', 'Global', 'GoSub', 'GoTo',
+                'Handles', 'If', 'Implements', 'Inherits', 'Interface', 'Let',
+                'Lib', 'Loop', 'Me', 'MustInherit', 'MustOverride', 'MyBase',
+                'MyClass', 'Narrowing', 'New', 'Next', 'Not', 'Nothing',
+                'NotInheritable', 'NotOverridable', 'Of', 'On', 'Operator',
+                'Option', 'Optional', 'Overloads', 'Overridable', 'Overrides',
+                'ParamArray', 'Partial', 'Private', 'Protected', 'Public',
+                'RaiseEvent', 'ReadOnly', 'ReDim', 'RemoveHandler', 'Resume',
+                'Return', 'Select', 'Set', 'Shadows', 'Shared', 'Single',
+                'Static', 'Step', 'Stop', 'SyncLock', 'Then', 'Throw', 'To',
+                'True', 'Try', 'TryCast', 'Wend', 'Using', 'When', 'While',
+                'Widening', 'With', 'WithEvents', 'WriteOnly'),
+                   prefix='(?<!\.)', suffix=r'\b'), Keyword),
             (r'(?<!\.)End\b', Keyword, 'end'),
             (r'(?<!\.)(Dim|Const)\b', Keyword, 'dim'),
             (r'(?<!\.)(Function|Sub|Property)(\s+)',
@@ -497,7 +498,7 @@ class GenericAspxLexer(RegexLexer):
 # TODO support multiple languages within the same source file
 class CSharpAspxLexer(DelegatingLexer):
     """
-    Lexer for highligting C# within ASP.NET pages.
+    Lexer for highlighting C# within ASP.NET pages.
     """
 
     name = 'aspx-cs'
@@ -518,7 +519,7 @@ class CSharpAspxLexer(DelegatingLexer):
 
 class VbNetAspxLexer(DelegatingLexer):
     """
-    Lexer for highligting Visual Basic.net within ASP.NET pages.
+    Lexer for highlighting Visual Basic.net within ASP.NET pages.
     """
 
     name = 'aspx-vb'
@@ -623,6 +624,7 @@ class FSharpLexer(RegexLexer):
             (r'\b(member|override)(\s+)(\w+)(\.)(\w+)',
              bygroups(Keyword, Text, Name, Punctuation, Name.Function)),
             (r'\b(%s)\b' % '|'.join(keywords), Keyword),
+            (r'``([^`\n\r\t]|`[^`\n\r\t])+``', Name),
             (r'(%s)' % '|'.join(keyopts), Operator),
             (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator),
             (r'\b(%s)\b' % '|'.join(word_operators), Operator.Word),

packages/wakatime/packages/pygments/lexers/dsls.py

@@ -5,18 +5,20 @@
 
     Lexers for various domain-specific languages.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import re
 
-from pygments.lexer import RegexLexer, bygroups, words, include, default
+from pygments.lexer import ExtendedRegexLexer, RegexLexer, bygroups, words, \
+    include, default, this, using, combined
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
-    Number, Punctuation, Literal
+    Number, Punctuation, Literal, Whitespace
 
 __all__ = ['ProtoBufLexer', 'BroLexer', 'PuppetLexer', 'RslLexer',
-           'MscgenLexer', 'VGLLexer', 'AlloyLexer', 'PanLexer']
+           'MscgenLexer', 'VGLLexer', 'AlloyLexer', 'PanLexer',
+           'CrmshLexer', 'ThriftLexer', 'FlatlineLexer', 'SnowballLexer']
 
 
 class ProtoBufLexer(RegexLexer):
@@ -34,7 +36,7 @@ class ProtoBufLexer(RegexLexer):
     tokens = {
         'root': [
             (r'[ \t]+', Text),
-            (r'[,;{}\[\]()]', Punctuation),
+            (r'[,;{}\[\]()<>]', Punctuation),
             (r'/(\\\n)?/(\n|(.|\n)*?[^\\]\n)', Comment.Single),
             (r'/(\\\n)?\*(.|\n)*?\*(\\\n)?/', Comment.Multiline),
             (words((
@@ -81,6 +83,111 @@ class ProtoBufLexer(RegexLexer):
     }
 
 
+class ThriftLexer(RegexLexer):
+    """
+    For `Thrift <https://thrift.apache.org/>`__ interface definitions.
+
+    .. versionadded:: 2.1
+    """
+    name = 'Thrift'
+    aliases = ['thrift']
+    filenames = ['*.thrift']
+    mimetypes = ['application/x-thrift']
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            include('comments'),
+            (r'"', String.Double, combined('stringescape', 'dqs')),
+            (r'\'', String.Single, combined('stringescape', 'sqs')),
+            (r'(namespace)(\s+)',
+                bygroups(Keyword.Namespace, Text.Whitespace), 'namespace'),
+            (r'(enum|union|struct|service|exception)(\s+)',
+                bygroups(Keyword.Declaration, Text.Whitespace), 'class'),
+            (r'((?:(?:[^\W\d]|\$)[\w.\[\]$<>]*\s+)+?)'  # return arguments
+             r'((?:[^\W\d]|\$)[\w$]*)'                  # method name
+             r'(\s*)(\()',                              # signature start
+             bygroups(using(this), Name.Function, Text, Operator)),
+            include('keywords'),
+            include('numbers'),
+            (r'[&=]', Operator),
+            (r'[:;,{}()<>\[\]]', Punctuation),
+            (r'[a-zA-Z_](\.\w|\w)*', Name),
+        ],
+        'whitespace': [
+            (r'\n', Text.Whitespace),
+            (r'\s+', Text.Whitespace),
+        ],
+        'comments': [
+            (r'#.*$', Comment),
+            (r'//.*?\n', Comment),
+            (r'/\*[\w\W]*?\*/', Comment.Multiline),
+        ],
+        'stringescape': [
+            (r'\\([\\nrt"\'])', String.Escape),
+        ],
+        'dqs': [
+            (r'"', String.Double, '#pop'),
+            (r'[^\\"\n]+', String.Double),
+        ],
+        'sqs': [
+            (r"'", String.Single, '#pop'),
+            (r'[^\\\'\n]+', String.Single),
+        ],
+        'namespace': [
+            (r'[a-z*](\.\w|\w)*', Name.Namespace, '#pop'),
+            default('#pop'),
+        ],
+        'class': [
+            (r'[a-zA-Z_]\w*', Name.Class, '#pop'),
+            default('#pop'),
+        ],
+        'keywords': [
+            (r'(async|oneway|extends|throws|required|optional)\b', Keyword),
+            (r'(true|false)\b', Keyword.Constant),
+            (r'(const|typedef)\b', Keyword.Declaration),
+            (words((
+                'cpp_namespace', 'cpp_include', 'cpp_type', 'java_package',
+                'cocoa_prefix', 'csharp_namespace', 'delphi_namespace',
+                'php_namespace', 'py_module', 'perl_package',
+                'ruby_namespace', 'smalltalk_category', 'smalltalk_prefix',
+                'xsd_all', 'xsd_optional', 'xsd_nillable', 'xsd_namespace',
+                'xsd_attrs', 'include'), suffix=r'\b'),
+             Keyword.Namespace),
+            (words((
+                'void', 'bool', 'byte', 'i16', 'i32', 'i64', 'double',
+                'string', 'binary', 'map', 'list', 'set', 'slist',
+                'senum'), suffix=r'\b'),
+             Keyword.Type),
+            (words((
+                'BEGIN', 'END', '__CLASS__', '__DIR__', '__FILE__',
+                '__FUNCTION__', '__LINE__', '__METHOD__', '__NAMESPACE__',
+                'abstract', 'alias', 'and', 'args', 'as', 'assert', 'begin',
+                'break', 'case', 'catch', 'class', 'clone', 'continue',
+                'declare', 'def', 'default', 'del', 'delete', 'do', 'dynamic',
+                'elif', 'else', 'elseif', 'elsif', 'end', 'enddeclare',
+                'endfor', 'endforeach', 'endif', 'endswitch', 'endwhile',
+                'ensure', 'except', 'exec', 'finally', 'float', 'for',
+                'foreach', 'function', 'global', 'goto', 'if', 'implements',
+                'import', 'in', 'inline', 'instanceof', 'interface', 'is',
+                'lambda', 'module', 'native', 'new', 'next', 'nil', 'not',
+                'or', 'pass', 'public', 'print', 'private', 'protected',
+                'raise', 'redo', 'rescue', 'retry', 'register', 'return',
+                'self', 'sizeof', 'static', 'super', 'switch', 'synchronized',
+                'then', 'this', 'throw', 'transient', 'try', 'undef',
+                'unless', 'unsigned', 'until', 'use', 'var', 'virtual',
+                'volatile', 'when', 'while', 'with', 'xor', 'yield'),
+                prefix=r'\b', suffix=r'\b'),
+             Keyword.Reserved),
+        ],
+        'numbers': [
+            (r'[+-]?(\d+\.\d+([eE][+-]?\d+)?|\.?\d+[eE][+-]?\d+)', Number.Float),
+            (r'[+-]?0x[0-9A-Fa-f]+', Number.Hex),
+            (r'[+-]?[0-9]+', Number.Integer),
+        ],
+    }
+
+
 class BroLexer(RegexLexer):
     """
     For `Bro <http://bro-ids.org/>`_ scripts.
@@ -471,19 +578,22 @@ class PanLexer(RegexLexer):
         ],
         'basic': [
             (words((
-                'if', 'for', 'with', 'else', 'type', 'bind', 'while', 'valid', 'final', 'prefix',
-                'unique', 'object', 'foreach', 'include', 'template', 'function', 'variable',
-                'structure', 'extensible', 'declaration'), prefix=r'\b', suffix=r'\s*\b'),
+                'if', 'for', 'with', 'else', 'type', 'bind', 'while', 'valid', 'final',
+                'prefix', 'unique', 'object', 'foreach', 'include', 'template',
+                'function', 'variable', 'structure', 'extensible', 'declaration'),
+                prefix=r'\b', suffix=r'\s*\b'),
              Keyword),
             (words((
-                'file_contents', 'format', 'index', 'length', 'match', 'matches', 'replace',
-                'splice', 'split', 'substr', 'to_lowercase', 'to_uppercase', 'debug', 'error',
-                'traceback', 'deprecated', 'base64_decode', 'base64_encode', 'digest', 'escape',
-                'unescape', 'append', 'create', 'first', 'nlist', 'key', 'list', 'merge', 'next',
-                'prepend', 'is_boolean', 'is_defined', 'is_double', 'is_list', 'is_long',
-                'is_nlist', 'is_null', 'is_number', 'is_property', 'is_resource', 'is_string',
-                'to_boolean', 'to_double', 'to_long', 'to_string', 'clone', 'delete', 'exists',
-                'path_exists', 'if_exists', 'return', 'value'), prefix=r'\b', suffix=r'\s*\b'),
+                'file_contents', 'format', 'index', 'length', 'match', 'matches',
+                'replace', 'splice', 'split', 'substr', 'to_lowercase', 'to_uppercase',
+                'debug', 'error', 'traceback', 'deprecated', 'base64_decode',
+                'base64_encode', 'digest', 'escape', 'unescape', 'append', 'create',
+                'first', 'nlist', 'key', 'list', 'merge', 'next', 'prepend', 'is_boolean',
+                'is_defined', 'is_double', 'is_list', 'is_long', 'is_nlist', 'is_null',
+                'is_number', 'is_property', 'is_resource', 'is_string', 'to_boolean',
+                'to_double', 'to_long', 'to_string', 'clone', 'delete', 'exists',
+                'path_exists', 'if_exists', 'return', 'value'),
+                prefix=r'\b', suffix=r'\s*\b'),
              Name.Builtin),
             (r'#.*', Comment),
             (r'\\[\w\W]', String.Escape),
@@ -512,3 +622,257 @@ class PanLexer(RegexLexer):
             include('root'),
         ],
     }
+
+
+class CrmshLexer(RegexLexer):
+    """
+    Lexer for `crmsh <http://crmsh.github.io/>`_ configuration files
+    for Pacemaker clusters.
+
+    .. versionadded:: 2.1
+    """
+    name = 'Crmsh'
+    aliases = ['crmsh', 'pcmk']
+    filenames = ['*.crmsh', '*.pcmk']
+    mimetypes = []
+
+    elem = words((
+        'node', 'primitive', 'group', 'clone', 'ms', 'location',
+        'colocation', 'order', 'fencing_topology', 'rsc_ticket',
+        'rsc_template', 'property', 'rsc_defaults',
+        'op_defaults', 'acl_target', 'acl_group', 'user', 'role',
+        'tag'), suffix=r'(?![\w#$-])')
+    sub = words((
+        'params', 'meta', 'operations', 'op', 'rule',
+        'attributes', 'utilization'), suffix=r'(?![\w#$-])')
+    acl = words(('read', 'write', 'deny'), suffix=r'(?![\w#$-])')
+    bin_rel = words(('and', 'or'), suffix=r'(?![\w#$-])')
+    un_ops = words(('defined', 'not_defined'), suffix=r'(?![\w#$-])')
+    date_exp = words(('in_range', 'date', 'spec', 'in'), suffix=r'(?![\w#$-])')
+    acl_mod = (r'(?:tag|ref|reference|attribute|type|xpath)')
+    bin_ops = (r'(?:lt|gt|lte|gte|eq|ne)')
+    val_qual = (r'(?:string|version|number)')
+    rsc_role_action = (r'(?:Master|Started|Slave|Stopped|'
+                       r'start|promote|demote|stop)')
+
+    tokens = {
+        'root': [
+            (r'^#.*\n?', Comment),
+            # attr=value (nvpair)
+            (r'([\w#$-]+)(=)("(?:""|[^"])*"|\S+)',
+                bygroups(Name.Attribute, Punctuation, String)),
+            # need this construct, otherwise numeric node ids
+            # are matched as scores
+            # elem id:
+            (r'(node)(\s+)([\w#$-]+)(:)',
+                bygroups(Keyword, Whitespace, Name, Punctuation)),
+            # scores
+            (r'([+-]?([0-9]+|inf)):', Number),
+            # keywords (elements and other)
+            (elem, Keyword),
+            (sub, Keyword),
+            (acl, Keyword),
+            # binary operators
+            (r'(?:%s:)?(%s)(?![\w#$-])' % (val_qual, bin_ops), Operator.Word),
+            # other operators
+            (bin_rel, Operator.Word),
+            (un_ops, Operator.Word),
+            (date_exp, Operator.Word),
+            # builtin attributes (e.g. #uname)
+            (r'#[a-z]+(?![\w#$-])', Name.Builtin),
+            # acl_mod:blah
+            (r'(%s)(:)("(?:""|[^"])*"|\S+)' % acl_mod,
+             bygroups(Keyword, Punctuation, Name)),
+            # rsc_id[:(role|action)]
+            # NB: this matches all other identifiers
+            (r'([\w#$-]+)(?:(:)(%s))?(?![\w#$-])' % rsc_role_action,
+             bygroups(Name, Punctuation, Operator.Word)),
+            # punctuation
+            (r'(\\(?=\n)|[[\](){}/:@])', Punctuation),
+            (r'\s+|\n', Whitespace),
+        ],
+    }
+
+
+class FlatlineLexer(RegexLexer):
+    """
+    Lexer for `Flatline <https://github.com/bigmlcom/flatline>`_ expressions.
+
+    .. versionadded:: 2.2
+    """
+    name = 'Flatline'
+    aliases = ['flatline']
+    filenames = []
+    mimetypes = ['text/x-flatline']
+
+    special_forms = ('let',)
+
+    builtins = (
+        "!=", "*", "+", "-", "<", "<=", "=", ">", ">=", "abs", "acos", "all",
+        "all-but", "all-with-defaults", "all-with-numeric-default", "and",
+        "asin", "atan", "avg", "avg-window", "bin-center", "bin-count", "call",
+        "category-count", "ceil", "cond", "cond-window", "cons", "cos", "cosh",
+        "count", "diff-window", "div", "ensure-value", "ensure-weighted-value",
+        "epoch", "epoch-day", "epoch-fields", "epoch-hour", "epoch-millisecond",
+        "epoch-minute", "epoch-month", "epoch-second", "epoch-weekday",
+        "epoch-year", "exp", "f", "field", "field-prop", "fields", "filter",
+        "first", "floor", "head", "if", "in", "integer", "language", "length",
+        "levenshtein", "linear-regression", "list", "ln", "log", "log10", "map",
+        "matches", "matches?", "max", "maximum", "md5", "mean", "median", "min",
+        "minimum", "missing", "missing-count", "missing?", "missing_count",
+        "mod", "mode", "normalize", "not", "nth", "occurrences", "or",
+        "percentile", "percentile-label", "population", "population-fraction",
+        "pow", "preferred", "preferred?", "quantile-label", "rand", "rand-int",
+        "random-value", "re-quote", "real", "replace", "replace-first", "rest",
+        "round", "row-number", "segment-label", "sha1", "sha256", "sin", "sinh",
+        "sqrt", "square", "standard-deviation", "standard_deviation", "str",
+        "subs", "sum", "sum-squares", "sum-window", "sum_squares", "summary",
+        "summary-no", "summary-str", "tail", "tan", "tanh", "to-degrees",
+        "to-radians", "variance", "vectorize", "weighted-random-value", "window",
+        "winnow", "within-percentiles?", "z-score",
+    )
+
+    valid_name = r'(?!#)[\w!$%*+<=>?/.#-]+'
+
+    tokens = {
+        'root': [
+            # whitespaces - usually not relevant
+            (r'[,\s]+', Text),
+
+            # numbers
+            (r'-?\d+\.\d+', Number.Float),
+            (r'-?\d+', Number.Integer),
+            (r'0x-?[a-f\d]+', Number.Hex),
+
+            # strings, symbols and characters
+            (r'"(\\\\|\\"|[^"])*"', String),
+            (r"\\(.|[a-z]+)", String.Char),
+
+            # expression template placeholder
+            (r'_', String.Symbol),
+
+            # highlight the special forms
+            (words(special_forms, suffix=' '), Keyword),
+
+            # highlight the builtins
+            (words(builtins, suffix=' '), Name.Builtin),
+
+            # the remaining functions
+            (r'(?<=\()' + valid_name, Name.Function),
+
+            # find the remaining variables
+            (valid_name, Name.Variable),
+
+            # parentheses
+            (r'(\(|\))', Punctuation),
+        ],
+    }
+
+
+class SnowballLexer(ExtendedRegexLexer):
+    """
+    Lexer for `Snowball <http://snowballstem.org/>`_ source code.
+
+    .. versionadded:: 2.2
+    """
+
+    name = 'Snowball'
+    aliases = ['snowball']
+    filenames = ['*.sbl']
+
+    _ws = r'\n\r\t '
+
+    def __init__(self, **options):
+        self._reset_stringescapes()
+        ExtendedRegexLexer.__init__(self, **options)
+
+    def _reset_stringescapes(self):
+        self._start = "'"
+        self._end = "'"
+
+    def _string(do_string_first):
+        def callback(lexer, match, ctx):
+            s = match.start()
+            text = match.group()
+            string = re.compile(r'([^%s]*)(.)' % re.escape(lexer._start)).match
+            escape = re.compile(r'([^%s]*)(.)' % re.escape(lexer._end)).match
+            pos = 0
+            do_string = do_string_first
+            while pos < len(text):
+                if do_string:
+                    match = string(text, pos)
+                    yield s + match.start(1), String.Single, match.group(1)
+                    if match.group(2) == "'":
+                        yield s + match.start(2), String.Single, match.group(2)
+                        ctx.stack.pop()
+                        break
+                    yield s + match.start(2), String.Escape, match.group(2)
+                    pos = match.end()
+                match = escape(text, pos)
+                yield s + match.start(), String.Escape, match.group()
+                if match.group(2) != lexer._end:
+                    ctx.stack[-1] = 'escape'
+                    break
+                pos = match.end()
+                do_string = True
+            ctx.pos = s + match.end()
+        return callback
+
+    def _stringescapes(lexer, match, ctx):
+        lexer._start = match.group(3)
+        lexer._end = match.group(5)
+        return bygroups(Keyword.Reserved, Text, String.Escape, Text,
+                        String.Escape)(lexer, match, ctx)
+
+    tokens = {
+        'root': [
+            (words(('len', 'lenof'), suffix=r'\b'), Operator.Word),
+            include('root1'),
+        ],
+        'root1': [
+            (r'[%s]+' % _ws, Text),
+            (r'\d+', Number.Integer),
+            (r"'", String.Single, 'string'),
+            (r'[()]', Punctuation),
+            (r'/\*[\w\W]*?\*/', Comment.Multiline),
+            (r'//.*', Comment.Single),
+            (r'[!*+\-/<=>]=|[-=]>|<[+-]|[$*+\-/<=>?\[\]]', Operator),
+            (words(('as', 'get', 'hex', 'among', 'define', 'decimal',
+                    'backwardmode'), suffix=r'\b'),
+             Keyword.Reserved),
+            (words(('strings', 'booleans', 'integers', 'routines', 'externals',
+                    'groupings'), suffix=r'\b'),
+             Keyword.Reserved, 'declaration'),
+            (words(('do', 'or', 'and', 'for', 'hop', 'non', 'not', 'set', 'try',
+                    'fail', 'goto', 'loop', 'next', 'test', 'true',
+                    'false', 'unset', 'atmark', 'attach', 'delete', 'gopast',
+                    'insert', 'repeat', 'sizeof', 'tomark', 'atleast',
+                    'atlimit', 'reverse', 'setmark', 'tolimit', 'setlimit',
+                    'backwards', 'substring'), suffix=r'\b'),
+             Operator.Word),
+            (words(('size', 'limit', 'cursor', 'maxint', 'minint'),
+                   suffix=r'\b'),
+             Name.Builtin),
+            (r'(stringdef\b)([%s]*)([^%s]+)' % (_ws, _ws),
+             bygroups(Keyword.Reserved, Text, String.Escape)),
+            (r'(stringescapes\b)([%s]*)(.)([%s]*)(.)' % (_ws, _ws),
+             _stringescapes),
+            (r'[A-Za-z]\w*', Name),
+        ],
+        'declaration': [
+            (r'\)', Punctuation, '#pop'),
+            (words(('len', 'lenof'), suffix=r'\b'), Name,
+             ('root1', 'declaration')),
+            include('root1'),
+        ],
+        'string': [
+            (r"[^']*'", _string(True)),
+        ],
+        'escape': [
+            (r"[^']*'", _string(False)),
+        ],
+    }
+
+    def get_tokens_unprocessed(self, text=None, context=None):
+        self._reset_stringescapes()
+        return ExtendedRegexLexer.get_tokens_unprocessed(self, text, context)

packages/wakatime/packages/pygments/lexers/dylan.py

@@ -5,7 +5,7 @@
 
     Lexers for the Dylan language.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/ecl.py

@@ -5,7 +5,7 @@
 
     Lexers for the ECL language.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/eiffel.py

@@ -5,7 +5,7 @@
 
     Lexer for the Eiffel language.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/elm.py

@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.elm
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexer for the Elm programming language.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, words, include
+from pygments.token import Comment, Keyword, Name, Number, Punctuation, String, Text
+
+__all__ = ['ElmLexer']
+
+
+class ElmLexer(RegexLexer):
+    """
+    For `Elm <http://elm-lang.org/>`_ source code.
+
+    .. versionadded:: 2.1
+    """
+
+    name = 'Elm'
+    aliases = ['elm']
+    filenames = ['*.elm']
+    mimetypes = ['text/x-elm']
+
+    validName = r'[a-z_][a-zA-Z_\']*'
+
+    specialName = r'^main '
+
+    builtinOps = (
+        '~', '||', '|>', '|', '`', '^', '\\', '\'', '>>', '>=', '>', '==',
+        '=', '<~', '<|', '<=', '<<', '<-', '<', '::', ':', '/=', '//', '/',
+        '..', '.', '->', '-', '++', '+', '*', '&&', '%',
+    )
+
+    reservedWords = words((
+        'alias', 'as', 'case', 'else', 'if', 'import', 'in',
+        'let', 'module', 'of', 'port', 'then', 'type', 'where',
+        ), suffix=r'\b')
+
+    tokens = {
+        'root': [
+
+            # Comments
+            (r'\{-', Comment.Multiline, 'comment'),
+            (r'--.*', Comment.Single),
+
+            # Whitespace
+            (r'\s+', Text),
+
+            # Strings
+            (r'"', String, 'doublequote'),
+
+            # Modules
+            (r'^\s*module\s*', Keyword.Namespace, 'imports'),
+
+            # Imports
+            (r'^\s*import\s*', Keyword.Namespace, 'imports'),
+
+            # Shaders
+            (r'\[glsl\|.*', Name.Entity, 'shader'),
+
+            # Keywords
+            (reservedWords, Keyword.Reserved),
+
+            # Types
+            (r'[A-Z]\w*', Keyword.Type),
+
+            # Main
+            (specialName, Keyword.Reserved),
+
+            # Prefix Operators
+            (words((builtinOps), prefix=r'\(', suffix=r'\)'), Name.Function),
+
+            # Infix Operators
+            (words((builtinOps)), Name.Function),
+
+            # Numbers
+            include('numbers'),
+
+            # Variable Names
+            (validName, Name.Variable),
+
+            # Parens
+            (r'[,()\[\]{}]', Punctuation),
+
+        ],
+
+        'comment': [
+            (r'-(?!\})', Comment.Multiline),
+            (r'\{-', Comment.Multiline, 'comment'),
+            (r'[^-}]', Comment.Multiline),
+            (r'-\}', Comment.Multiline, '#pop'),
+        ],
+
+        'doublequote': [
+            (r'\\u[0-9a-fA-F]{4}', String.Escape),
+            (r'\\[nrfvb\\"]', String.Escape),
+            (r'[^"]', String),
+            (r'"', String, '#pop'),
+        ],
+
+        'imports': [
+            (r'\w+(\.\w+)*', Name.Class, '#pop'),
+        ],
+
+        'numbers': [
+            (r'_?\d+\.(?=\d+)', Number.Float),
+            (r'_?\d+', Number.Integer),
+        ],
+
+        'shader': [
+            (r'\|(?!\])', Name.Entity),
+            (r'\|\]', Name.Entity, '#pop'),
+            (r'.*\n', Name.Entity),
+        ],
+    }

packages/wakatime/packages/pygments/lexers/erlang.py

@@ -5,7 +5,7 @@
 
     Lexers for Erlang.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -82,7 +82,11 @@ class ErlangLexer(RegexLexer):
 
     variable_re = r'(?:[A-Z_]\w*)'
 
-    escape_re = r'(?:\\(?:[bdefnrstv\'"\\/]|[0-7][0-7]?[0-7]?|\^[a-zA-Z]))'
+    esc_char_re = r'[bdefnrstv\'"\\]'
+    esc_octal_re = r'[0-7][0-7]?[0-7]?'
+    esc_hex_re = r'(?:x[0-9a-fA-F]{2}|x\{[0-9a-fA-F]+\})'
+    esc_ctrl_re = r'\^[a-zA-Z]'
+    escape_re = r'(?:\\(?:'+esc_char_re+r'|'+esc_octal_re+r'|'+esc_hex_re+r'|'+esc_ctrl_re+r'))'
 
     macro_re = r'(?:'+variable_re+r'|'+atom_re+r')'
 
@@ -112,11 +116,18 @@ class ErlangLexer(RegexLexer):
             (r'\?'+macro_re, Name.Constant),
             (r'\$(?:'+escape_re+r'|\\[ %]|[^\\])', String.Char),
             (r'#'+atom_re+r'(:?\.'+atom_re+r')?', Name.Label),
+
+            # Erlang script shebang
+            (r'\A#!.+\n', Comment.Hashbang),
+
+            # EEP 43: Maps
+            # http://www.erlang.org/eeps/eep-0043.html
+            (r'#\{', Punctuation, 'map_key'),
         ],
         'string': [
             (escape_re, String.Escape),
             (r'"', String, '#pop'),
-            (r'~[0-9.*]*[~#+bBcdefginpPswWxX]', String.Interpol),
+            (r'~[0-9.*]*[~#+BPWXb-ginpswx]', String.Interpol),
             (r'[^"\\~]+', String),
             (r'~', String),
         ],
@@ -127,6 +138,17 @@ class ErlangLexer(RegexLexer):
              bygroups(Name.Entity, Text, Punctuation, Name.Label), '#pop'),
             (atom_re, Name.Entity, '#pop'),
         ],
+        'map_key': [
+            include('root'),
+            (r'=>', Punctuation, 'map_val'),
+            (r':=', Punctuation, 'map_val'),
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'map_val': [
+            include('root'),
+            (r',', Punctuation, '#pop'),
+            (r'(?=\})', Punctuation, '#pop'),
+        ],
     }
 
 
@@ -218,11 +240,11 @@ class ElixirLexer(RegexLexer):
     KEYWORD_OPERATOR = ('not', 'and', 'or', 'when', 'in')
     BUILTIN = (
         'case', 'cond', 'for', 'if', 'unless', 'try', 'receive', 'raise',
-        'quote', 'unquote', 'unquote_splicing', 'throw', 'super'
+        'quote', 'unquote', 'unquote_splicing', 'throw', 'super',
     )
     BUILTIN_DECLARATION = (
         'def', 'defp', 'defmodule', 'defprotocol', 'defmacro', 'defmacrop',
-        'defdelegate', 'defexception', 'defstruct', 'defimpl', 'defcallback'
+        'defdelegate', 'defexception', 'defstruct', 'defimpl', 'defcallback',
     )
 
     BUILTIN_NAMESPACE = ('import', 'require', 'use', 'alias')
@@ -241,7 +263,7 @@ class ElixirLexer(RegexLexer):
     OPERATORS1 = ('<', '>', '+', '-', '*', '/', '!', '^', '&')
 
     PUNCTUATION = (
-        '\\\\', '<<', '>>', '=>', '(', ')', ':', ';', ',', '[', ']'
+        '\\\\', '<<', '>>', '=>', '(', ')', ':', ';', ',', '[', ']',
     )
 
     def get_tokens_unprocessed(self, text):

packages/wakatime/packages/pygments/lexers/esoteric.py

@@ -0,0 +1,277 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.esoteric
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for esoteric languages.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Error
+
+__all__ = ['BrainfuckLexer', 'BefungeLexer', 'RedcodeLexer', 'CAmkESLexer',
+           'CapDLLexer', 'AheuiLexer']
+
+
+class BrainfuckLexer(RegexLexer):
+    """
+    Lexer for the esoteric `BrainFuck <http://www.muppetlabs.com/~breadbox/bf/>`_
+    language.
+    """
+
+    name = 'Brainfuck'
+    aliases = ['brainfuck', 'bf']
+    filenames = ['*.bf', '*.b']
+    mimetypes = ['application/x-brainfuck']
+
+    tokens = {
+        'common': [
+            # use different colors for different instruction types
+            (r'[.,]+', Name.Tag),
+            (r'[+-]+', Name.Builtin),
+            (r'[<>]+', Name.Variable),
+            (r'[^.,+\-<>\[\]]+', Comment),
+        ],
+        'root': [
+            (r'\[', Keyword, 'loop'),
+            (r'\]', Error),
+            include('common'),
+        ],
+        'loop': [
+            (r'\[', Keyword, '#push'),
+            (r'\]', Keyword, '#pop'),
+            include('common'),
+        ]
+    }
+
+
+class BefungeLexer(RegexLexer):
+    """
+    Lexer for the esoteric `Befunge <http://en.wikipedia.org/wiki/Befunge>`_
+    language.
+
+    .. versionadded:: 0.7
+    """
+    name = 'Befunge'
+    aliases = ['befunge']
+    filenames = ['*.befunge']
+    mimetypes = ['application/x-befunge']
+
+    tokens = {
+        'root': [
+            (r'[0-9a-f]', Number),
+            (r'[+*/%!`-]', Operator),             # Traditional math
+            (r'[<>^v?\[\]rxjk]', Name.Variable),  # Move, imperatives
+            (r'[:\\$.,n]', Name.Builtin),         # Stack ops, imperatives
+            (r'[|_mw]', Keyword),
+            (r'[{}]', Name.Tag),                  # Befunge-98 stack ops
+            (r'".*?"', String.Double),            # Strings don't appear to allow escapes
+            (r'\'.', String.Single),              # Single character
+            (r'[#;]', Comment),                   # Trampoline... depends on direction hit
+            (r'[pg&~=@iotsy]', Keyword),          # Misc
+            (r'[()A-Z]', Comment),                # Fingerprints
+            (r'\s+', Text),                       # Whitespace doesn't matter
+        ],
+    }
+
+
+class CAmkESLexer(RegexLexer):
+    """
+    Basic lexer for the input language for the
+    `CAmkES <https://sel4.systems/CAmkES/>`_ component platform.
+
+    .. versionadded:: 2.1
+    """
+    name = 'CAmkES'
+    aliases = ['camkes', 'idl4']
+    filenames = ['*.camkes', '*.idl4']
+
+    tokens = {
+        'root': [
+            # C pre-processor directive
+            (r'^\s*#.*\n', Comment.Preproc),
+
+            # Whitespace, comments
+            (r'\s+', Text),
+            (r'/\*(.|\n)*?\*/', Comment),
+            (r'//.*\n', Comment),
+
+            (r'[\[(){},.;\]]', Punctuation),
+            (r'[~!%^&*+=|?:<>/-]', Operator),
+
+            (words(('assembly', 'attribute', 'component', 'composition',
+                    'configuration', 'connection', 'connector', 'consumes',
+                    'control', 'dataport', 'Dataport', 'Dataports', 'emits',
+                    'event', 'Event', 'Events', 'export', 'from', 'group',
+                    'hardware', 'has', 'interface', 'Interface', 'maybe',
+                    'procedure', 'Procedure', 'Procedures', 'provides',
+                    'template', 'thread', 'threads', 'to', 'uses', 'with'),
+                   suffix=r'\b'), Keyword),
+
+            (words(('bool', 'boolean', 'Buf', 'char', 'character', 'double',
+                    'float', 'in', 'inout', 'int', 'int16_6', 'int32_t',
+                    'int64_t', 'int8_t', 'integer', 'mutex', 'out', 'real',
+                    'refin', 'semaphore', 'signed', 'string', 'struct',
+                    'uint16_t', 'uint32_t', 'uint64_t', 'uint8_t', 'uintptr_t',
+                    'unsigned', 'void'),
+                   suffix=r'\b'), Keyword.Type),
+
+            # Recognised attributes
+            (r'[a-zA-Z_]\w*_(priority|domain|buffer)', Keyword.Reserved),
+            (words(('dma_pool', 'from_access', 'to_access'), suffix=r'\b'),
+                Keyword.Reserved),
+
+            # CAmkES-level include
+            (r'import\s+(<[^>]*>|"[^"]*");', Comment.Preproc),
+
+            # C-level include
+            (r'include\s+(<[^>]*>|"[^"]*");', Comment.Preproc),
+
+            # Literals
+            (r'0[xX][\da-fA-F]+', Number.Hex),
+            (r'-?[\d]+', Number),
+            (r'-?[\d]+\.[\d]+', Number.Float),
+            (r'"[^"]*"', String),
+            (r'[Tt]rue|[Ff]alse', Name.Builtin),
+
+            # Identifiers
+            (r'[a-zA-Z_]\w*', Name),
+        ],
+    }
+
+
+class CapDLLexer(RegexLexer):
+    """
+    Basic lexer for
+    `CapDL <https://ssrg.nicta.com.au/publications/nictaabstracts/Kuz_KLW_10.abstract.pml>`_.
+
+    The source of the primary tool that reads such specifications is available
+    at https://github.com/seL4/capdl/tree/master/capDL-tool. Note that this
+    lexer only supports a subset of the grammar. For example, identifiers can
+    shadow type names, but these instances are currently incorrectly
+    highlighted as types. Supporting this would need a stateful lexer that is
+    considered unnecessarily complex for now.
+
+    .. versionadded:: 2.2
+    """
+    name = 'CapDL'
+    aliases = ['capdl']
+    filenames = ['*.cdl']
+
+    tokens = {
+        'root': [
+            # C pre-processor directive
+            (r'^\s*#.*\n', Comment.Preproc),
+
+            # Whitespace, comments
+            (r'\s+', Text),
+            (r'/\*(.|\n)*?\*/', Comment),
+            (r'(//|--).*\n', Comment),
+
+            (r'[<>\[(){},:;=\]]', Punctuation),
+            (r'\.\.', Punctuation),
+
+            (words(('arch', 'arm11', 'caps', 'child_of', 'ia32', 'irq', 'maps',
+                    'objects'), suffix=r'\b'), Keyword),
+
+            (words(('aep', 'asid_pool', 'cnode', 'ep', 'frame', 'io_device',
+                    'io_ports', 'io_pt', 'notification', 'pd', 'pt', 'tcb',
+                    'ut', 'vcpu'), suffix=r'\b'), Keyword.Type),
+
+            # Properties
+            (words(('asid', 'addr', 'badge', 'cached', 'dom', 'domainID', 'elf',
+                    'fault_ep', 'G', 'guard', 'guard_size', 'init', 'ip',
+                    'prio', 'sp', 'R', 'RG', 'RX', 'RW', 'RWG', 'RWX', 'W',
+                    'WG', 'WX', 'level', 'masked', 'master_reply', 'paddr',
+                    'ports', 'reply', 'uncached'), suffix=r'\b'),
+             Keyword.Reserved),
+
+            # Literals
+            (r'0[xX][\da-fA-F]+', Number.Hex),
+            (r'\d+(\.\d+)?(k|M)?', Number),
+            (words(('bits',), suffix=r'\b'), Number),
+            (words(('cspace', 'vspace', 'reply_slot', 'caller_slot',
+                    'ipc_buffer_slot'), suffix=r'\b'), Number),
+
+            # Identifiers
+            (r'[a-zA-Z_][-@\.\w]*', Name),
+        ],
+    }
+
+
+class RedcodeLexer(RegexLexer):
+    """
+    A simple Redcode lexer based on ICWS'94.
+    Contributed by Adam Blinkinsop <blinks@acm.org>.
+
+    .. versionadded:: 0.8
+    """
+    name = 'Redcode'
+    aliases = ['redcode']
+    filenames = ['*.cw']
+
+    opcodes = ('DAT', 'MOV', 'ADD', 'SUB', 'MUL', 'DIV', 'MOD',
+               'JMP', 'JMZ', 'JMN', 'DJN', 'CMP', 'SLT', 'SPL',
+               'ORG', 'EQU', 'END')
+    modifiers = ('A', 'B', 'AB', 'BA', 'F', 'X', 'I')
+
+    tokens = {
+        'root': [
+            # Whitespace:
+            (r'\s+', Text),
+            (r';.*$', Comment.Single),
+            # Lexemes:
+            #  Identifiers
+            (r'\b(%s)\b' % '|'.join(opcodes), Name.Function),
+            (r'\b(%s)\b' % '|'.join(modifiers), Name.Decorator),
+            (r'[A-Za-z_]\w+', Name),
+            #  Operators
+            (r'[-+*/%]', Operator),
+            (r'[#$@<>]', Operator),  # mode
+            (r'[.,]', Punctuation),  # mode
+            #  Numbers
+            (r'[-+]?\d+', Number.Integer),
+        ],
+    }
+
+
+class AheuiLexer(RegexLexer):
+    """
+    Aheui_ Lexer.
+
+    Aheui_ is esoteric language based on Korean alphabets.
+
+    .. _Aheui:: http://aheui.github.io/
+
+    """
+
+    name = 'Aheui'
+    aliases = ['aheui']
+    filenames = ['*.aheui']
+
+    tokens = {
+        'root': [
+            (u'['
+             u'나-낳냐-냫너-넣녀-녛노-놓뇨-눟뉴-닇'
+             u'다-닿댜-댷더-덯뎌-뎧도-돟됴-둫듀-딓'
+             u'따-땋땨-떃떠-떻뗘-뗳또-똫뚀-뚷뜌-띟'
+             u'라-랗랴-럏러-렇려-렿로-롷료-뤃류-릫'
+             u'마-맣먀-먛머-멓며-몋모-뫃묘-뭏뮤-믷'
+             u'바-밯뱌-뱧버-벟벼-볗보-봏뵤-붛뷰-빃'
+             u'빠-빻뺘-뺳뻐-뻫뼈-뼣뽀-뽛뾰-뿧쀼-삏'
+             u'사-샇샤-샿서-섷셔-셯소-솧쇼-숳슈-싛'
+             u'싸-쌓쌰-썋써-쎃쎠-쎻쏘-쏳쑈-쑿쓔-씧'
+             u'자-잫쟈-쟣저-젛져-졓조-좋죠-줗쥬-즿'
+             u'차-챃챠-챻처-첳쳐-쳫초-촣쵸-춯츄-칗'
+             u'카-캏캬-컇커-컿켜-켷코-콯쿄-쿻큐-킣'
+             u'타-탛탸-턓터-텋텨-톃토-톻툐-퉇튜-틯'
+             u'파-팧퍄-퍟퍼-펗펴-폏포-퐇표-풓퓨-픻'
+             u'하-핳햐-햫허-헣혀-혛호-홓효-훟휴-힇'
+             u']', Operator),
+            ('.', Comment),
+        ],
+    }

packages/wakatime/packages/pygments/lexers/ezhil.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.ezhil
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Pygments lexers for Ezhil language.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Keyword, Text, Comment, Name
+from pygments.token import String, Number, Punctuation, Operator
+
+__all__ = ['EzhilLexer']
+
+
+class EzhilLexer(RegexLexer):
+    """
+    Lexer for `Ezhil, a Tamil script-based programming language <http://ezhillang.org>`_
+
+    .. versionadded:: 2.1
+    """
+    name = 'Ezhil'
+    aliases = ['ezhil']
+    filenames = ['*.n']
+    mimetypes = ['text/x-ezhil']
+    flags = re.MULTILINE | re.UNICODE
+    # Refer to tamil.utf8.tamil_letters from open-tamil for a stricter version of this.
+    # This much simpler version is close enough, and includes combining marks.
+    _TALETTERS = u'[a-zA-Z_]|[\u0b80-\u0bff]'
+    tokens = {
+        'root': [
+            include('keywords'),
+            (r'#.*\n', Comment.Single),
+            (r'[@+/*,^\-%]|[!<>=]=?|&&?|\|\|?', Operator),
+            (u'இல்', Operator.Word),
+            (words((u'assert', u'max', u'min',
+                    u'நீளம்', u'சரம்_இடமாற்று', u'சரம்_கண்டுபிடி',
+                    u'பட்டியல்', u'பின்இணை', u'வரிசைப்படுத்து',
+                    u'எடு', u'தலைகீழ்', u'நீட்டிக்க', u'நுழைக்க', u'வை',
+                    u'கோப்பை_திற', u'கோப்பை_எழுது', u'கோப்பை_மூடு',
+                    u'pi', u'sin', u'cos', u'tan', u'sqrt', u'hypot', u'pow',
+                    u'exp', u'log', u'log10', u'exit',
+                    ), suffix=r'\b'), Name.Builtin),
+            (r'(True|False)\b', Keyword.Constant),
+            (r'[^\S\n]+', Text),
+            include('identifier'),
+            include('literal'),
+            (r'[(){}\[\]:;.]', Punctuation),
+        ],
+        'keywords': [
+            (u'பதிப்பி|தேர்ந்தெடு|தேர்வு|ஏதேனில்|ஆனால்|இல்லைஆனால்|இல்லை|ஆக|ஒவ்வொன்றாக|இல்|வரை|செய்|முடியேனில்|பின்கொடு|முடி|நிரல்பாகம்|தொடர்|நிறுத்து|நிரல்பாகம்', Keyword),
+        ],
+        'identifier': [
+            (u'(?:'+_TALETTERS+u')(?:[0-9]|'+_TALETTERS+u')*', Name),
+        ],
+        'literal': [
+            (r'".*?"', String),
+            (r'(?u)\d+((\.\d*)?[eE][+-]?\d+|\.\d*)', Number.Float),
+            (r'(?u)\d+', Number.Integer),
+        ]
+    }
+
+    def __init__(self, **options):
+        super(EzhilLexer, self).__init__(**options)
+        self.encoding = options.get('encoding', 'utf-8')

packages/wakatime/packages/pygments/lexers/factor.py

@@ -5,7 +5,7 @@
 
     Lexers for the Factor language.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/fantom.py

@@ -5,7 +5,7 @@
 
     Lexer for the Fantom language.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/lexers/felix.py

@@ -5,7 +5,7 @@
 
     Lexer for the Felix language.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -237,7 +237,7 @@ class FelixLexer(RegexLexer):
         ],
         'strings': [
             (r'%(\([a-zA-Z0-9]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
-             '[hlL]?[diouxXeEfFgGcrs%]', String.Interpol),
+             '[hlL]?[E-GXc-giorsux%]', String.Interpol),
             (r'[^\\\'"%\n]+', String),
             # quotes, percents and backslashes must be parsed one at a time
             (r'[\'"\\]', String),

packages/wakatime/packages/pygments/lexers/forth.py

@@ -0,0 +1,177 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.forth
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups
+from pygments.token import Error, Punctuation, Literal, Token, \
+     Text, Comment, Operator, Keyword, Name, String, Number, Generic
+
+
+__all__ = ['ForthLexer']
+
+
+class ForthLexer(RegexLexer):
+    """
+    Lexer for Forth files.
+
+    .. versionadded:: 2.2
+    """
+    name = 'Forth'
+    aliases = ['forth']
+    filenames = ['*.frt', '*.fs']
+    mimetypes = ['application/x-forth']
+
+    delimiter = r'\s'
+    delimiter_end = r'(?=[%s])' % delimiter
+
+    valid_name_chars = r'[^%s]' % delimiter
+    valid_name = r"%s+%s" % (valid_name_chars, delimiter_end)
+
+    flags = re.IGNORECASE | re.MULTILINE
+
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+            # All comment types
+            (r'\\.*?\n', Comment.Single),
+            (r'\([\s].*?\)', Comment.Single),
+            # defining words. The next word is a new command name
+            (r'(:|variable|constant|value|buffer:)(\s+)',
+             bygroups(Keyword.Namespace, Text), 'worddef'),
+            # strings are rather simple
+            (r'([.sc]")(\s+?)', bygroups(String, Text), 'stringdef'),
+            # keywords from the various wordsets
+            # *** Wordset BLOCK
+            (r'(blk|block|buffer|evaluate|flush|load|save-buffers|update|'
+             # *** Wordset BLOCK-EXT
+             r'empty-buffers|list|refill|scr|thru|'
+             # *** Wordset CORE
+             r'\#s|\*\/mod|\+loop|\/mod|0<|0=|1\+|1-|2!|'
+             r'2\*|2\/|2@|2drop|2dup|2over|2swap|>body|'
+             r'>in|>number|>r|\?dup|abort|abort\"|abs|'
+             r'accept|align|aligned|allot|and|base|begin|'
+             r'bl|c!|c,|c@|cell\+|cells|char|char\+|'
+             r'chars|constant|count|cr|create|decimal|'
+             r'depth|do|does>|drop|dup|else|emit|environment\?|'
+             r'evaluate|execute|exit|fill|find|fm\/mod|'
+             r'here|hold|i|if|immediate|invert|j|key|'
+             r'leave|literal|loop|lshift|m\*|max|min|'
+             r'mod|move|negate|or|over|postpone|quit|'
+             r'r>|r@|recurse|repeat|rot|rshift|s\"|s>d|'
+             r'sign|sm\/rem|source|space|spaces|state|swap|'
+             r'then|type|u\.|u\<|um\*|um\/mod|unloop|until|'
+             r'variable|while|word|xor|\[char\]|\[\'\]|'
+             r'@|!|\#|<\#|\#>|:|;|\+|-|\*|\/|,|<|>|\|1\+|1-|\.|'
+            # *** Wordset CORE-EXT
+             r'\.r|0<>|'
+             r'0>|2>r|2r>|2r@|:noname|\?do|again|c\"|'
+             r'case|compile,|endcase|endof|erase|false|'
+             r'hex|marker|nip|of|pad|parse|pick|refill|'
+             r'restore-input|roll|save-input|source-id|to|'
+             r'true|tuck|u\.r|u>|unused|value|within|'
+             r'\[compile\]|'
+            # *** Wordset CORE-EXT-obsolescent
+             r'\#tib|convert|expect|query|span|'
+             r'tib|'
+            # *** Wordset DOUBLE
+             r'2constant|2literal|2variable|d\+|d-|'
+             r'd\.|d\.r|d0<|d0=|d2\*|d2\/|d<|d=|d>s|'
+             r'dabs|dmax|dmin|dnegate|m\*\/|m\+|'
+            # *** Wordset DOUBLE-EXT
+             r'2rot|du<|'
+            # *** Wordset EXCEPTION
+             r'catch|throw|'
+            # *** Wordset EXCEPTION-EXT
+             r'abort|abort\"|'
+            # *** Wordset FACILITY
+             r'at-xy|key\?|page|'
+            # *** Wordset FACILITY-EXT
+             r'ekey|ekey>char|ekey\?|emit\?|ms|time&date|'
+            # *** Wordset FILE
+             r'BIN|CLOSE-FILE|CREATE-FILE|DELETE-FILE|FILE-POSITION|'
+             r'FILE-SIZE|INCLUDE-FILE|INCLUDED|OPEN-FILE|R\/O|'
+             r'R\/W|READ-FILE|READ-LINE|REPOSITION-FILE|RESIZE-FILE|'
+             r'S\"|SOURCE-ID|W/O|WRITE-FILE|WRITE-LINE|'
+            # *** Wordset FILE-EXT
+             r'FILE-STATUS|FLUSH-FILE|REFILL|RENAME-FILE|'
+            # *** Wordset FLOAT
+             r'>float|d>f|'
+             r'f!|f\*|f\+|f-|f\/|f0<|f0=|f<|f>d|f@|'
+             r'falign|faligned|fconstant|fdepth|fdrop|fdup|'
+             r'fliteral|float\+|floats|floor|fmax|fmin|'
+             r'fnegate|fover|frot|fround|fswap|fvariable|'
+             r'represent|'
+            # *** Wordset FLOAT-EXT
+             r'df!|df@|dfalign|dfaligned|dfloat\+|'
+             r'dfloats|f\*\*|f\.|fabs|facos|facosh|falog|'
+             r'fasin|fasinh|fatan|fatan2|fatanh|fcos|fcosh|'
+             r'fe\.|fexp|fexpm1|fln|flnp1|flog|fs\.|fsin|'
+             r'fsincos|fsinh|fsqrt|ftan|ftanh|f~|precision|'
+             r'set-precision|sf!|sf@|sfalign|sfaligned|sfloat\+|'
+             r'sfloats|'
+            # *** Wordset LOCAL
+             r'\(local\)|to|'
+            # *** Wordset LOCAL-EXT
+             r'locals\||'
+            # *** Wordset MEMORY
+             r'allocate|free|resize|'
+            # *** Wordset SEARCH
+             r'definitions|find|forth-wordlist|get-current|'
+             r'get-order|search-wordlist|set-current|set-order|'
+             r'wordlist|'
+            # *** Wordset SEARCH-EXT
+             r'also|forth|only|order|previous|'
+            # *** Wordset STRING
+             r'-trailing|\/string|blank|cmove|cmove>|compare|'
+             r'search|sliteral|'
+            # *** Wordset TOOLS
+             r'.s|dump|see|words|'
+            # *** Wordset TOOLS-EXT
+             r';code|'
+             r'ahead|assembler|bye|code|cs-pick|cs-roll|'
+             r'editor|state|\[else\]|\[if\]|\[then\]|'
+            # *** Wordset TOOLS-EXT-obsolescent
+            r'forget|'
+            # Forth 2012
+            r'defer|defer@|defer!|action-of|begin-structure|field:|buffer:|'
+            r'parse-name|buffer:|traverse-wordlist|n>r|nr>|2value|fvalue|'
+            r'name>interpret|name>compile|name>string|'
+            r'cfield:|end-structure)'+delimiter, Keyword),
+
+            # Numbers
+            (r'(\$[0-9A-F]+)', Number.Hex),
+            (r'(\#|%|&|\-|\+)?[0-9]+', Number.Integer),
+            (r'(\#|%|&|\-|\+)?[0-9.]+', Keyword.Type),
+            # amforth specific
+            (r'(@i|!i|@e|!e|pause|noop|turnkey|sleep|'
+             r'itype|icompare|sp@|sp!|rp@|rp!|up@|up!|'
+             r'>a|a>|a@|a!|a@+|a@-|>b|b>|b@|b!|b@+|b@-|'
+             r'find-name|1ms|'
+             r'sp0|rp0|\(evaluate\)|int-trap|int!)' + delimiter,
+             Name.Constant),
+            # a proposal
+            (r'(do-recognizer|r:fail|recognizer:|get-recognizers|'
+             r'set-recognizers|r:float|r>comp|r>int|r>post|'
+             r'r:name|r:word|r:dnum|r:num|recognizer|forth-recognizer|'
+             r'rec:num|rec:float|rec:word)' + delimiter, Name.Decorator),
+            # defining words. The next word is a new command name
+            (r'(Evalue|Rvalue|Uvalue|Edefer|Rdefer|Udefer)(\s+)',
+             bygroups(Keyword.Namespace, Text), 'worddef'),
+
+            (valid_name, Name.Function),      # Anything else is executed
+
+        ],
+        'worddef': [
+            (r'\S+', Name.Class, '#pop'),
+        ],
+        'stringdef': [
+            (r'[^"]+', String, '#pop'),
+        ],
+    }

packages/wakatime/packages/pygments/lexers/fortran.py

@@ -5,17 +5,17 @@
 
     Lexers for Fortran languages.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import re
 
-from pygments.lexer import RegexLexer, include, words
+from pygments.lexer import RegexLexer, bygroups, include, words, using, default
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
-    Number, Punctuation
+    Number, Punctuation, Generic
 
-__all__ = ['FortranLexer']
+__all__ = ['FortranLexer', 'FortranFixedLexer']
 
 
 class FortranLexer(RegexLexer):
@@ -26,7 +26,7 @@ class FortranLexer(RegexLexer):
     """
     name = 'Fortran'
     aliases = ['fortran']
-    filenames = ['*.f', '*.f90', '*.F', '*.F90']
+    filenames = ['*.f03', '*.f90', '*.F03', '*.F90']
     mimetypes = ['text/x-fortran']
     flags = re.IGNORECASE | re.MULTILINE
 
@@ -73,13 +73,14 @@ class FortranLexer(RegexLexer):
             # Data Types
             (words((
                 'CHARACTER', 'COMPLEX', 'DOUBLE PRECISION', 'DOUBLE COMPLEX', 'INTEGER',
-                'LOGICAL', 'REAL', 'C_INT', 'C_SHORT', 'C_LONG', 'C_LONG_LONG', 'C_SIGNED_CHAR',
-                'C_SIZE_T', 'C_INT8_T', 'C_INT16_T', 'C_INT32_T', 'C_INT64_T', 'C_INT_LEAST8_T',
-                'C_INT_LEAST16_T', 'C_INT_LEAST32_T', 'C_INT_LEAST64_T', 'C_INT_FAST8_T',
-                'C_INT_FAST16_T', 'C_INT_FAST32_T', 'C_INT_FAST64_T', 'C_INTMAX_T',
-                'C_INTPTR_T', 'C_FLOAT', 'C_DOUBLE', 'C_LONG_DOUBLE', 'C_FLOAT_COMPLEX',
-                'C_DOUBLE_COMPLEX', 'C_LONG_DOUBLE_COMPLEX', 'C_BOOL', 'C_CHAR', 'C_PTR',
-                'C_FUNPTR'), prefix=r'\b', suffix=r'\s*\b'),
+                'LOGICAL', 'REAL', 'C_INT', 'C_SHORT', 'C_LONG', 'C_LONG_LONG',
+                'C_SIGNED_CHAR', 'C_SIZE_T', 'C_INT8_T', 'C_INT16_T', 'C_INT32_T',
+                'C_INT64_T', 'C_INT_LEAST8_T', 'C_INT_LEAST16_T', 'C_INT_LEAST32_T',
+                'C_INT_LEAST64_T', 'C_INT_FAST8_T', 'C_INT_FAST16_T', 'C_INT_FAST32_T',
+                'C_INT_FAST64_T', 'C_INTMAX_T', 'C_INTPTR_T', 'C_FLOAT', 'C_DOUBLE',
+                'C_LONG_DOUBLE', 'C_FLOAT_COMPLEX', 'C_DOUBLE_COMPLEX',
+                'C_LONG_DOUBLE_COMPLEX', 'C_BOOL', 'C_CHAR', 'C_PTR', 'C_FUNPTR'),
+                   prefix=r'\b', suffix=r'\s*\b'),
              Keyword.Type),
 
             # Operators
@@ -155,7 +156,50 @@ class FortranLexer(RegexLexer):
 
         'nums': [
             (r'\d+(?![.e])(_[a-z]\w+)?', Number.Integer),
-            (r'[+-]?\d*\.\d+(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float),
-            (r'[+-]?\d+\.\d*(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float),
+            (r'[+-]?\d*\.\d+([ed][-+]?\d+)?(_[a-z]\w+)?', Number.Float),
+            (r'[+-]?\d+\.\d*([ed][-+]?\d+)?(_[a-z]\w+)?', Number.Float),
         ],
     }
+
+
+class FortranFixedLexer(RegexLexer):
+    """
+    Lexer for fixed format Fortran.
+
+    .. versionadded:: 2.1
+    """
+    name = 'FortranFixed'
+    aliases = ['fortranfixed']
+    filenames = ['*.f', '*.F']
+
+    flags = re.IGNORECASE
+
+    def _lex_fortran(self, match, ctx=None):
+        """Lex a line just as free form fortran without line break."""
+        lexer = FortranLexer()
+        text = match.group(0) + "\n"
+        for index, token, value in lexer.get_tokens_unprocessed(text):
+            value = value.replace('\n', '')
+            if value != '':
+                yield index, token, value
+
+    tokens = {
+        'root': [
+            (r'[C*].*\n', Comment),
+            (r'#.*\n', Comment.Preproc),
+            (r' {0,4}!.*\n', Comment),
+            (r'(.{5})', Name.Label, 'cont-char'),
+            (r'.*\n', using(FortranLexer)),
+        ],
+        'cont-char': [
+            (' ', Text, 'code'),
+            ('0', Comment, 'code'),
+            ('.', Generic.Strong, 'code'),
+        ],
+        'code': [
+            (r'(.{66})(.*)(\n)',
+             bygroups(_lex_fortran, Comment, Text), 'root'),
+            (r'(.*)(\n)', bygroups(_lex_fortran, Text), 'root'),
+            default('root'),
+        ]
+    }

packages/wakatime/packages/pygments/lexers/foxpro.py

@@ -5,7 +5,7 @@
 
     Simple lexer for Microsoft Visual FoxPro source code.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """