v7.0.16

fix UnicodeDecodeError: 'ascii' codec can't decode byte 0xd0 in posiion 10: ordinal not in range(128) on plugin_loaded (ST2)

AUTHORS

@@ -11,5 +11,5 @@ Development Lead
 Patches and Suggestions
 -----------------------
 
-- 3onyc <3onyc@x3tech.com>
-- userid <xixico@ymail.com>
+- Jimmy Selgen Nielsen <jimmy.selgen@gmail.com>
+- Patrik Kernstock <info@pkern.at>

Default.sublime-commands

@@ -0,0 +1,6 @@
+[
+	{
+		"caption": "WakaTime: Open Dashboard",
+		"command": "wakatime_dashboard"
+	}
+]

HISTORY.rst

@@ -3,6 +3,652 @@ History
 -------
 
 
+7.0.16 (2017-02-20)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v7.0.2.
+
+
+7.0.15 (2017-02-13)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.2.2.
+- Upgrade pygments library to v2.2.0 for improved language detection.
+
+
+7.0.14 (2017-02-08)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.2.1.
+- Allow boolean or list of regex patterns for hidefilenames config setting.
+
+
+7.0.13 (2016-11-11)
+++++++++++++++++++
+
+- Support old Sublime Text with Python 2.6.
+- Fix bug that prevented reading default api key from existing config file.
+
+
+7.0.12 (2016-10-24)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.2.0.
+- Exit with status code 104 when api key is missing or invalid. Exit with
+  status code 103 when config file missing or invalid.
+- New WAKATIME_HOME env variable for setting path to config and log files.
+- Improve debug warning message from unsupported dependency parsers.
+
+
+7.0.11 (2016-09-23)
+++++++++++++++++++
+
+- Handle UnicodeDecodeError when when logging. Related to #68.
+
+
+7.0.10 (2016-09-22)
+++++++++++++++++++
+
+- Handle UnicodeDecodeError when looking for python. Fixes #68.
+- Upgrade wakatime-cli to v6.0.9.
+
+
+7.0.9 (2016-09-02)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.0.8.
+
+
+7.0.8 (2016-07-21)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to master version to fix debug logging encoding bug.
+
+
+7.0.7 (2016-07-06)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.0.7.
+- Handle unknown exceptions from requests library by deleting cached session
+  object because it could be from a previous conflicting version.
+- New hostname setting in config file to set machine hostname. Hostname
+  argument takes priority over hostname from config file.
+- Prevent logging unrelated exception when logging tracebacks.
+- Use correct namespace for pygments.lexers.ClassNotFound exception so it is
+  caught when dependency detection not available for a language.
+
+
+7.0.6 (2016-06-13)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.0.5.
+- Upgrade pygments to v2.1.3 for better language coverage.
+
+
+7.0.5 (2016-06-08)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to master version to fix bug in urllib3 package causing
+  unhandled retry exceptions.
+- Prevent tracking git branch with detached head.
+
+
+7.0.4 (2016-05-21)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.0.3.
+- Upgrade requests dependency to v2.10.0.
+- Support for SOCKS proxies.
+
+
+7.0.3 (2016-05-16)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.0.2.
+- Prevent popup on Mac when xcode-tools is not installed.
+
+
+7.0.2 (2016-04-29)
+++++++++++++++++++
+
+- Prevent implicit unicode decoding from string format when logging output
+  from Python version check.
+
+
+7.0.1 (2016-04-28)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v6.0.1.
+- Fix bug which prevented plugin from being sent with extra heartbeats.
+
+
+7.0.0 (2016-04-28)
+++++++++++++++++++
+
+- Queue heartbeats and send to wakatime-cli after 4 seconds.
+- Nest settings menu under Package Settings.
+- Upgrade wakatime-cli to v6.0.0.
+- Increase default network timeout to 60 seconds when sending heartbeats to
+  the api.
+- New --extra-heartbeats command line argument for sending a JSON array of
+  extra queued heartbeats to STDIN.
+- Change --entitytype command line argument to --entity-type.
+- No longer allowing --entity-type of url.
+- Support passing an alternate language to cli to be used when a language can
+  not be guessed from the code file.
+
+
+6.0.8 (2016-04-18)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v5.0.0.
+- Support regex patterns in projectmap config section for renaming projects.
+- Upgrade pytz to v2016.3.
+- Upgrade tzlocal to v1.2.2.
+
+
+6.0.7 (2016-03-11)
+++++++++++++++++++
+
+- Fix bug causing RuntimeError when finding Python location
+
+
+6.0.6 (2016-03-06)
+++++++++++++++++++
+
+- upgrade wakatime-cli to v4.1.13
+- encode TimeZone as utf-8 before adding to headers
+- encode X-Machine-Name as utf-8 before adding to headers
+
+
+6.0.5 (2016-03-06)
+++++++++++++++++++
+
+- upgrade wakatime-cli to v4.1.11
+- encode machine hostname as Unicode when adding to X-Machine-Name header
+
+
+6.0.4 (2016-01-15)
+++++++++++++++++++
+
+- fix UnicodeDecodeError on ST2 with non-English locale
+
+
+6.0.3 (2016-01-11)
+++++++++++++++++++
+
+- upgrade wakatime-cli core to v4.1.10
+- accept 201 or 202 response codes as success from api
+- upgrade requests package to v2.9.1
+
+
+6.0.2 (2016-01-06)
+++++++++++++++++++
+
+- upgrade wakatime-cli core to v4.1.9
+- improve C# dependency detection
+- correctly log exception tracebacks
+- log all unknown exceptions to wakatime.log file
+- disable urllib3 SSL warning from every request
+- detect dependencies from golang files
+- use api.wakatime.com for sending heartbeats
+
+
+6.0.1 (2016-01-01)
+++++++++++++++++++
+
+- use embedded python if system python is broken, or doesn't output a version number
+- log output from wakatime-cli in ST console when in debug mode
+
+
+6.0.0 (2015-12-01)
+++++++++++++++++++
+
+- use embeddable Python instead of installing on Windows
+
+
+5.0.1 (2015-10-06)
+++++++++++++++++++
+
+- look for python in system PATH again
+
+
+5.0.0 (2015-10-02)
+++++++++++++++++++
+
+- improve logging with levels and log function
+- switch registry warnings to debug log level
+
+
+4.0.20 (2015-10-01)
+++++++++++++++++++
+
+- correctly find python binary in non-Windows environments
+
+
+4.0.19 (2015-10-01)
+++++++++++++++++++
+
+- handle case where ST builtin python does not have _winreg or winreg module
+
+
+4.0.18 (2015-10-01)
+++++++++++++++++++
+
+- find python location from windows registry
+
+
+4.0.17 (2015-10-01)
+++++++++++++++++++
+
+- download python in non blocking background thread for Windows machines
+
+
+4.0.16 (2015-09-29)
+++++++++++++++++++
+
+- upgrade wakatime cli to v4.1.8
+- fix bug in guess_language function
+- improve dependency detection
+- default request timeout of 30 seconds
+- new --timeout command line argument to change request timeout in seconds
+- allow passing command line arguments using sys.argv
+- fix entry point for pypi distribution
+- new --entity and --entitytype command line arguments
+
+
+4.0.15 (2015-08-28)
+++++++++++++++++++
+
+- upgrade wakatime cli to v4.1.3
+- fix local session caching
+
+
+4.0.14 (2015-08-25)
+++++++++++++++++++
+
+- upgrade wakatime cli to v4.1.2
+- fix bug in offline caching which prevented heartbeats from being cleaned up
+
+
+4.0.13 (2015-08-25)
+++++++++++++++++++
+
+- upgrade wakatime cli to v4.1.1
+- send hostname in X-Machine-Name header
+- catch exceptions from pygments.modeline.get_filetype_from_buffer
+- upgrade requests package to v2.7.0
+- handle non-ASCII characters in import path on Windows, won't fix for Python2
+- upgrade argparse to v1.3.0
+- move language translations to api server
+- move extension rules to api server
+- detect correct header file language based on presence of .cpp or .c files named the same as the .h file
+
+
+4.0.12 (2015-07-31)
+++++++++++++++++++
+
+- correctly use urllib in Python3
+
+
+4.0.11 (2015-07-31)
+++++++++++++++++++
+
+- install python if missing on Windows OS
+
+
+4.0.10 (2015-07-31)
+++++++++++++++++++
+
+- downgrade requests library to v2.6.0
+
+
+4.0.9 (2015-07-29)
+++++++++++++++++++
+
+- catch exceptions from pygments.modeline.get_filetype_from_buffer
+
+
+4.0.8 (2015-06-23)
+++++++++++++++++++
+
+- fix offline logging
+- limit language detection to known file extensions, unless file contents has a vim modeline
+- upgrade wakatime cli to v4.0.16
+
+
+4.0.7 (2015-06-21)
+++++++++++++++++++
+
+- allow customizing status bar message in sublime-settings file
+- guess language using multiple methods, then use most accurate guess
+- use entity and type for new heartbeats api resource schema
+- correctly log message from py.warnings module
+- upgrade wakatime cli to v4.0.15
+
+
+4.0.6 (2015-05-16)
+++++++++++++++++++
+
+- fix bug with auto detecting project name
+- upgrade wakatime cli to v4.0.13
+
+
+4.0.5 (2015-05-15)
+++++++++++++++++++
+
+- correctly display caller and lineno in log file when debug is true
+- project passed with --project argument will always be used
+- new --alternate-project argument
+- upgrade wakatime cli to v4.0.12
+
+
+4.0.4 (2015-05-12)
+++++++++++++++++++
+
+- reuse SSL connection over multiple processes for improved performance
+- upgrade wakatime cli to v4.0.11
+
+
+4.0.3 (2015-05-06)
+++++++++++++++++++
+
+- send cursorpos to wakatime cli
+- upgrade wakatime cli to v4.0.10
+
+
+4.0.2 (2015-05-06)
+++++++++++++++++++
+
+- only send heartbeats for the currently active buffer
+
+
+4.0.1 (2015-05-06)
+++++++++++++++++++
+
+- ignore git temporary files
+- don't send two write heartbeats within 2 seconds of eachother
+
+
+4.0.0 (2015-04-12)
+++++++++++++++++++
+
+- listen for selection modified instead of buffer activated for better performance
+
+
+3.0.19 (2015-04-07)
++++++++++++++++++++
+
+- fix bug in project detection when folder not found
+
+
+3.0.18 (2015-04-04)
++++++++++++++++++++
+
+- upgrade wakatime cli to v4.0.8
+- added api_url config option to .wakatime.cfg file
+
+
+3.0.17 (2015-04-02)
++++++++++++++++++++
+
+- use open folder as current project when not using revision control
+
+
+3.0.16 (2015-04-02)
++++++++++++++++++++
+
+- copy list when obfuscating api key so original command is not modified
+
+
+3.0.15 (2015-04-01)
++++++++++++++++++++
+
+- obfuscate api key when logging to Sublime Text Console in debug mode
+
+
+3.0.14 (2015-03-31)
++++++++++++++++++++
+
+- always use external python binary because ST builtin python does not support checking SSL certs
+- upgrade wakatime cli to v4.0.6
+
+
+3.0.13 (2015-03-23)
++++++++++++++++++++
+
+- correctly check for SSL support in ST built-in python
+- fix status bar message
+
+
+3.0.12 (2015-03-23)
++++++++++++++++++++
+
+- always use unicode function from compat module when encoding log messages
+
+
+3.0.11 (2015-03-23)
++++++++++++++++++++
+
+- upgrade simplejson package to v3.6.5
+
+
+3.0.10 (2015-03-22)
++++++++++++++++++++
+
+- ability to disable status bar message from WakaTime.sublime-settings file
+
+
+3.0.9 (2015-03-20)
+++++++++++++++++++
+
+- status bar message showing when WakaTime plugin is enabled
+- moved some logic into thread to help prevent slow plugin warning message
+
+
+3.0.8 (2015-03-09)
+++++++++++++++++++
+
+- upgrade wakatime cli to v4.0.4
+- use requests library instead of urllib2, so api SSL cert is verified
+- new --notfile argument to support logging time without a real file
+- new --proxy argument for https proxy support
+- new options for excluding and including directories
+
+
+3.0.7 (2015-02-05)
+++++++++++++++++++
+
+- handle errors encountered when looking for .sublime-project file
+
+
+3.0.6 (2015-01-13)
+++++++++++++++++++
+
+- upgrade external wakatime package to v3.0.5
+- ignore errors from malformed markup (too many closing tags)
+
+
+3.0.5 (2015-01-06)
+++++++++++++++++++
+
+- upgrade external wakatime package to v3.0.4
+- remove unused dependency, which is missing in some python environments
+
+
+3.0.4 (2014-12-26)
+++++++++++++++++++
+
+- fix bug causing plugin to not work in Sublime Text 2
+
+
+3.0.3 (2014-12-25)
+++++++++++++++++++
+
+- upgrade external wakatime package to v3.0.3
+- detect JavaScript frameworks from script tags in Html template files
+
+
+3.0.2 (2014-12-25)
+++++++++++++++++++
+
+- upgrade external wakatime package to v3.0.2
+- detect frameworks from JavaScript and JSON files
+
+
+3.0.1 (2014-12-23)
+++++++++++++++++++
+
+- parse use namespaces from php files
+
+
+3.0.0 (2014-12-23)
+++++++++++++++++++
+
+- upgrade external wakatime package to v3.0.1
+- detect libraries and frameworks for C++, Java, .NET, PHP, and Python files
+
+
+2.0.21 (2014-12-22)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.1.11
+- fix bug in offline logging when no response from api
+
+
+2.0.20 (2014-12-05)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.1.9
+- fix bug preventing offline heartbeats from being purged after uploaded
+
+
+2.0.19 (2014-12-04)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.1.8
+- fix UnicodeDecodeError when building user agent string
+- handle case where response is None
+
+
+2.0.18 (2014-11-30)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.1.7
+- upgrade pygments to v2.0.1
+- always log an error when api key is incorrect
+
+
+2.0.17 (2014-11-18)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.1.6
+- fix list index error when detecting subversion project
+
+
+2.0.16 (2014-11-12)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.1.4
+- when Python was not compiled with https support, log an error to the log file
+
+
+2.0.15 (2014-11-10)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.1.3
+- correctly detect branch for subversion projects
+
+
+2.0.14 (2014-10-14)
+++++++++++++++++++
+
+- popup error message if Python binary not found
+
+
+2.0.13 (2014-10-07)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.1.2
+- still log heartbeat when something goes wrong while reading num lines in file
+
+
+2.0.12 (2014-09-30)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.1.1
+- fix bug where binary file opened as utf-8
+
+
+2.0.11 (2014-09-30)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.1.0
+- python3 compatibility changes
+
+
+2.0.10 (2014-08-29)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.0.8
+- supress output from svn command
+
+
+2.0.9 (2014-08-27)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.0.7
+- fix support for subversion projects on Mac OS X
+
+
+2.0.8 (2014-08-07)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.0.6
+- fix unicode bug by encoding json POST data
+
+
+2.0.7 (2014-07-25)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.0.5
+- option in .wakatime.cfg to obfuscate file names
+
+
+2.0.6 (2014-07-25)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.0.4
+- use unique logger namespace to prevent collisions in shared plugin environments
+
+
+2.0.5 (2014-06-18)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.0.3
+- use project name from sublime-project file when no revision control project found
+
+
+2.0.4 (2014-06-09)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.0.2
+- disable offline logging when Python not compiled with sqlite3 module
+
+
+2.0.3 (2014-05-26)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.0.1
+- fix bug in queue preventing completed tasks from being purged
+
+
+2.0.2 (2014-05-26)
+++++++++++++++++++
+
+- disable syncing offline time until bug fixed
+
+
 2.0.1 (2014-05-25)
 ++++++++++++++++++
 

LICENSE

@@ -1,4 +1,6 @@
-Copyright (c) 2014 Alan Hamlett https://wakatime.com
+BSD 3-Clause License
+
+Copyright (c) 2014 by the respective authors (see AUTHORS file).
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -12,7 +14,7 @@ modification, are permitted provided that the following conditions are met:
   in the documentation and/or other materials provided
   with the distribution.
 
-* Neither the names of Wakatime or WakaTime, nor the names of its
+* Neither the names of WakaTime, nor the names of its
   contributors may be used to endorse or promote products derived
   from this software without specific prior written permission.
 

Main.sublime-menu

@@ -6,24 +6,37 @@
         "children":
         [
             {
-                "caption": "WakaTime",
-                "mnemonic": "W",
-                "id": "wakatime-settings",
+                "caption": "Package Settings",
+                "mnemonic": "P",
+                "id": "package-settings",
                 "children":
                 [
                     {
-                        "command": "open_file", "args":
-                        {
-                            "file": "${packages}/WakaTime/WakaTime.sublime-settings"
-                        },
-                        "caption": "Settings – Default"
-                    },
-                    {
-                        "command": "open_file", "args":
-                        {
-                            "file": "${packages}/User/WakaTime.sublime-settings"
-                        },
-                        "caption": "Settings – User"
+                        "caption": "WakaTime",
+                        "mnemonic": "W",
+                        "id": "wakatime-settings",
+                        "children":
+                        [
+                            {
+                                "command": "open_file", "args":
+                                {
+                                    "file": "${packages}/WakaTime/WakaTime.sublime-settings"
+                                },
+                                "caption": "Settings – Default"
+                            },
+                            {
+                                "command": "open_file", "args":
+                                {
+                                    "file": "${packages}/User/WakaTime.sublime-settings"
+                                },
+                                "caption": "Settings – User"
+                            },
+                            {
+                                "command": "wakatime_dashboard",
+                                "args": {},
+                                "caption": "WakaTime Dashboard"
+                            }
+                        ]
                     }
                 ]
             }

README.md

@@ -1,33 +1,56 @@
 sublime-wakatime
 ================
 
-Fully automatic time tracking for Sublime Text 2 & 3.
+Metrics, insights, and time tracking automatically generated from your programming activity.
+
 
 Installation
 ------------
 
-Heads Up! For Sublime Text 2 on Windows & Linux, WakaTime depends on [Python](http://www.python.org/getit/) being installed to work correctly.
+1. Install [Package Control](https://packagecontrol.io/installation).
 
-1. Get an api key from: https://wakatime.com/#apikey
+2. Using [Package Control](https://packagecontrol.io/docs/usage):
 
-2. Using [Sublime Package Control](http://wbond.net/sublime_packages/package_control):
-
-  a) Press `ctrl+shift+p`(Windows, Linux) or `cmd+shift+p`(OS X).
+  a) Inside Sublime, press `ctrl+shift+p`(Windows, Linux) or `cmd+shift+p`(OS X).
 
   b) Type `install`, then press `enter` with `Package Control: Install Package` selected.
 
   c) Type `wakatime`, then press `enter` with the `WakaTime` plugin selected.
 
-3. You will see a prompt at the bottom asking for your [api key](https://wakatime.com/#apikey). Enter your api key, then press `enter`.
+3. Enter your [api key](https://wakatime.com/settings#apikey), then press `enter`.
 
-4. Use Sublime and your time will automatically be tracked for you.
+4. Use Sublime and your time will be tracked for you automatically.
 
-5. Visit https://wakatime.com to see your logged time.
+5. Visit https://wakatime.com/dashboard to see your logged time.
 
-6. Consider installing [BIND9](https://help.ubuntu.com/community/BIND9ServerHowto#Caching_Server_configuration) to cache your repeated DNS requests: `sudo apt-get install bind9`
 
 Screen Shots
 ------------
 
-![Project Overview](https://wakatime.com/static/img/ScreenShots/Screen Shot 2013-10-26 at 5.04.01 PM.png)
+![Project Overview](https://wakatime.com/static/img/ScreenShots/Screen-Shot-2016-03-21.png)
 
+
+Unresponsive Plugin Warning
+---------------------------
+
+In Sublime Text 2, if you get a warning message:
+
+    A plugin (WakaTime) may be making Sublime Text unresponsive by taking too long (0.017332s) in its on_modified callback.
+
+To fix this, go to `Preferences > Settings - User` then add the following setting:
+
+`"detect_slow_plugins": false`
+
+
+Troubleshooting
+---------------
+
+First, turn on debug mode in your `WakaTime.sublime-settings` file.
+
+![sublime user settings](https://wakatime.com/static/img/ScreenShots/sublime-wakatime-settings-menu.png?v=3)
+
+Add the line: `"debug": true`
+
+Then, open your Sublime Console with `View -> Show Console` to see the plugin executing the wakatime cli process when sending a heartbeat. Also, tail your `$HOME/.wakatime.log` file to debug wakatime cli problems.
+
+For more general troubleshooting information, see [wakatime/wakatime#troubleshooting](https://github.com/wakatime/wakatime#troubleshooting).

WakaTime.py

@@ -6,166 +6,641 @@ License:     BSD, see LICENSE for more details.
 Website:     https://wakatime.com/
 ==========================================================="""
 
-__version__ = '2.0.1'
+
+__version__ = '7.0.16'
+
 
 import sublime
 import sublime_plugin
 
-import glob
+import contextlib
+import json
 import os
 import platform
+import re
 import sys
 import time
 import threading
-import uuid
-from os.path import expanduser, dirname, realpath, isfile, join, exists
+import traceback
+import urllib
+import webbrowser
+from datetime import datetime
+from subprocess import Popen, STDOUT, PIPE
+from zipfile import ZipFile
+try:
+    import _winreg as winreg  # py2
+except ImportError:
+    try:
+        import winreg  # py3
+    except ImportError:
+        winreg = None
+try:
+    import Queue as queue  # py2
+except ImportError:
+    import queue  # py3
+
+
+is_py2 = (sys.version_info[0] == 2)
+is_py3 = (sys.version_info[0] == 3)
+
+if is_py2:
+    def u(text):
+        if text is None:
+            return None
+        if isinstance(text, unicode):
+            return text
+        try:
+            return text.decode('utf-8')
+        except:
+            try:
+                return text.decode(sys.getdefaultencoding())
+            except:
+                try:
+                    return unicode(text)
+                except:
+                    try:
+                        return text.decode('utf-8', 'replace')
+                    except:
+                        try:
+                            return unicode(str(text))
+                        except:
+                            return unicode('')
+
+elif is_py3:
+    def u(text):
+        if text is None:
+            return None
+        if isinstance(text, bytes):
+            try:
+                return text.decode('utf-8')
+            except:
+                try:
+                    return text.decode(sys.getdefaultencoding())
+                except:
+                    pass
+        try:
+            return str(text)
+        except:
+            return text.decode('utf-8', 'replace')
+
+else:
+    raise Exception('Unsupported Python version: {0}.{1}.{2}'.format(
+        sys.version_info[0],
+        sys.version_info[1],
+        sys.version_info[2],
+    ))
 
 
 # globals
-ACTION_FREQUENCY = 2
+HEARTBEAT_FREQUENCY = 2
 ST_VERSION = int(sublime.version())
-PLUGIN_DIR = dirname(realpath(__file__))
-API_CLIENT = '%s/packages/wakatime/wakatime-cli.py' % PLUGIN_DIR
+PLUGIN_DIR = os.path.dirname(os.path.realpath(__file__))
+API_CLIENT = os.path.join(PLUGIN_DIR, 'packages', 'wakatime', 'cli.py')
 SETTINGS_FILE = 'WakaTime.sublime-settings'
 SETTINGS = {}
-LAST_ACTION = {
+LAST_HEARTBEAT = {
     'time': 0,
     'file': None,
     'is_write': False,
 }
-HAS_SSL = False
-LOCK = threading.RLock()
+PYTHON_LOCATION = None
+HEARTBEATS = queue.Queue()
 
-# check if we have SSL support
+
+# Log Levels
+DEBUG = 'DEBUG'
+INFO = 'INFO'
+WARNING = 'WARNING'
+ERROR = 'ERROR'
+
+
+# add wakatime package to path
+sys.path.insert(0, os.path.join(PLUGIN_DIR, 'packages'))
 try:
-    import ssl
-    import socket
-    socket.ssl
-    HAS_SSL = True
-except (ImportError, AttributeError):
-    from subprocess import Popen
+    from wakatime.main import parseConfigFile
+except ImportError:
+    pass
 
-if HAS_SSL:
-    # import wakatime package
-    sys.path.insert(0, join(PLUGIN_DIR, 'packages', 'wakatime'))
-    import wakatime
+
+def set_timeout(callback, seconds):
+    """Runs the callback after the given seconds delay.
+
+    If this is Sublime Text 3, runs the callback on an alternate thread. If this
+    is Sublime Text 2, runs the callback in the main thread.
+    """
+
+    milliseconds = int(seconds * 1000)
+    try:
+        sublime.set_timeout_async(callback, milliseconds)
+    except AttributeError:
+        sublime.set_timeout(callback, milliseconds)
+
+
+def log(lvl, message, *args, **kwargs):
+    try:
+        if lvl == DEBUG and not SETTINGS.get('debug'):
+            return
+        msg = message
+        if len(args) > 0:
+            msg = message.format(*args)
+        elif len(kwargs) > 0:
+            msg = message.format(**kwargs)
+        try:
+            print('[WakaTime] [{lvl}] {msg}'.format(lvl=lvl, msg=msg))
+        except UnicodeDecodeError:
+            print(u('[WakaTime] [{lvl}] {msg}').format(lvl=lvl, msg=u(msg)))
+    except RuntimeError:
+        set_timeout(lambda: log(lvl, message, *args, **kwargs), 0)
+
+
+def resources_folder():
+    if platform.system() == 'Windows':
+        return os.path.join(os.getenv('APPDATA'), 'WakaTime')
+    else:
+        return os.path.join(os.path.expanduser('~'), '.wakatime')
+
+
+def update_status_bar(status):
+    """Updates the status bar."""
+
+    try:
+        if SETTINGS.get('status_bar_message'):
+            msg = datetime.now().strftime(SETTINGS.get('status_bar_message_fmt'))
+            if '{status}' in msg:
+                msg = msg.format(status=status)
+
+            active_window = sublime.active_window()
+            if active_window:
+                for view in active_window.views():
+                    view.set_status('wakatime', msg)
+    except RuntimeError:
+        set_timeout(lambda: update_status_bar(status), 0)
+
+
+def create_config_file():
+    """Creates the .wakatime.cfg INI file in $HOME directory, if it does
+    not already exist.
+    """
+    configFile = os.path.join(os.path.expanduser('~'), '.wakatime.cfg')
+    try:
+        with open(configFile) as fh:
+            pass
+    except IOError:
+        try:
+            with open(configFile, 'w') as fh:
+                fh.write("[settings]\n")
+                fh.write("debug = false\n")
+                fh.write("hidefilenames = false\n")
+        except IOError:
+            pass
 
 
 def prompt_api_key():
     global SETTINGS
-    if not SETTINGS.get('api_key'):
+
+    create_config_file()
+
+    default_key = ''
+    try:
+        configs = parseConfigFile()
+        if configs is not None:
+            if configs.has_option('settings', 'api_key'):
+                default_key = configs.get('settings', 'api_key')
+    except:
+        pass
+
+    if SETTINGS.get('api_key'):
+        return True
+    else:
         def got_key(text):
             if text:
                 SETTINGS.set('api_key', str(text))
                 sublime.save_settings(SETTINGS_FILE)
         window = sublime.active_window()
         if window:
-            window.show_input_panel('[WakaTime] Enter your wakatime.com api key:', '', got_key, None, None)
+            window.show_input_panel('[WakaTime] Enter your wakatime.com api key:', default_key, got_key, None, None)
             return True
         else:
-            print('[WakaTime] Error: Could not prompt for api key because no window found.')
+            log(ERROR, 'Could not prompt for api key because no window found.')
     return False
 
 
 def python_binary():
-    if platform.system() == 'Windows':
-        try:
-            Popen(['pythonw', '--version'])
-            return 'pythonw'
-        except:
-            for path in glob.iglob('/python*'):
-                if exists(realpath(join(path, 'pythonw.exe'))):
-                    return realpath(join(path, 'pythonw'))
-            return None
-    return 'python'
+    if PYTHON_LOCATION is not None:
+        return PYTHON_LOCATION
+
+    # look for python in PATH and common install locations
+    paths = [
+        os.path.join(resources_folder(), 'python'),
+        None,
+        '/',
+        '/usr/local/bin/',
+        '/usr/bin/',
+    ]
+    for path in paths:
+        path = find_python_in_folder(path)
+        if path is not None:
+            set_python_binary_location(path)
+            return path
+
+    # look for python in windows registry
+    path = find_python_from_registry(r'SOFTWARE\Python\PythonCore')
+    if path is not None:
+        set_python_binary_location(path)
+        return path
+    path = find_python_from_registry(r'SOFTWARE\Wow6432Node\Python\PythonCore')
+    if path is not None:
+        set_python_binary_location(path)
+        return path
+
+    return None
 
 
-def enough_time_passed(now, last_time):
-    if now - last_time > ACTION_FREQUENCY * 60:
+def set_python_binary_location(path):
+    global PYTHON_LOCATION
+    PYTHON_LOCATION = path
+    log(DEBUG, 'Found Python at: {0}'.format(path))
+
+
+def find_python_from_registry(location, reg=None):
+    if platform.system() != 'Windows' or winreg is None:
+        return None
+
+    if reg is None:
+        path = find_python_from_registry(location, reg=winreg.HKEY_CURRENT_USER)
+        if path is None:
+            path = find_python_from_registry(location, reg=winreg.HKEY_LOCAL_MACHINE)
+        return path
+
+    val = None
+    sub_key = 'InstallPath'
+    compiled = re.compile(r'^\d+\.\d+$')
+
+    try:
+        with winreg.OpenKey(reg, location) as handle:
+            versions = []
+            try:
+                for index in range(1024):
+                    version = winreg.EnumKey(handle, index)
+                    try:
+                        if compiled.search(version):
+                            versions.append(version)
+                    except re.error:
+                        pass
+            except EnvironmentError:
+                pass
+            versions.sort(reverse=True)
+            for version in versions:
+                try:
+                    path = winreg.QueryValue(handle, version + '\\' + sub_key)
+                    if path is not None:
+                        path = find_python_in_folder(path)
+                        if path is not None:
+                            log(DEBUG, 'Found python from {reg}\\{key}\\{version}\\{sub_key}.'.format(
+                                reg=reg,
+                                key=location,
+                                version=version,
+                                sub_key=sub_key,
+                            ))
+                            return path
+                except WindowsError:
+                    log(DEBUG, 'Could not read registry value "{reg}\\{key}\\{version}\\{sub_key}".'.format(
+                        reg=reg,
+                        key=location,
+                        version=version,
+                        sub_key=sub_key,
+                    ))
+    except WindowsError:
+        log(DEBUG, 'Could not read registry value "{reg}\\{key}".'.format(
+            reg=reg,
+            key=location,
+        ))
+    except:
+        log(ERROR, 'Could not read registry value "{reg}\\{key}":\n{exc}'.format(
+            reg=reg,
+            key=location,
+            exc=traceback.format_exc(),
+        ))
+
+    return val
+
+
+def find_python_in_folder(folder, headless=True):
+    pattern = re.compile(r'\d+\.\d+')
+
+    path = 'python'
+    if folder is not None:
+        path = os.path.realpath(os.path.join(folder, 'python'))
+    if headless:
+        path = u(path) + u('w')
+    log(DEBUG, u('Looking for Python at: {0}').format(u(path)))
+    try:
+        process = Popen([path, '--version'], stdout=PIPE, stderr=STDOUT)
+        output, err = process.communicate()
+        output = u(output).strip()
+        retcode = process.poll()
+        log(DEBUG, u('Python Version Output: {0}').format(output))
+        if not retcode and pattern.search(output):
+            return path
+    except:
+        log(DEBUG, u(sys.exc_info()[1]))
+
+    if headless:
+        path = find_python_in_folder(folder, headless=False)
+        if path is not None:
+            return path
+
+    return None
+
+
+def obfuscate_apikey(command_list):
+    cmd = list(command_list)
+    apikey_index = None
+    for num in range(len(cmd)):
+        if cmd[num] == '--key':
+            apikey_index = num + 1
+            break
+    if apikey_index is not None and apikey_index < len(cmd):
+        cmd[apikey_index] = 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXX' + cmd[apikey_index][-4:]
+    return cmd
+
+
+def enough_time_passed(now, is_write):
+    if now - LAST_HEARTBEAT['time'] > HEARTBEAT_FREQUENCY * 60:
+        return True
+    if is_write and now - LAST_HEARTBEAT['time'] > 2:
         return True
     return False
 
 
-def handle_action(view, is_write=False):
-    global LOCK, LAST_ACTION
-    with LOCK:
-        target_file = view.file_name()
-        if target_file:
-            thread = SendActionThread(target_file, is_write=is_write)
-            thread.start()
-            LAST_ACTION = {
-                'file': target_file,
-                'time': time.time(),
-                'is_write': is_write,
-            }
+def find_folder_containing_file(folders, current_file):
+    """Returns absolute path to folder containing the file.
+    """
+
+    parent_folder = None
+
+    current_folder = current_file
+    while True:
+        for folder in folders:
+            if os.path.realpath(os.path.dirname(current_folder)) == os.path.realpath(folder):
+                parent_folder = folder
+                break
+        if parent_folder is not None:
+            break
+        if not current_folder or os.path.dirname(current_folder) == current_folder:
+            break
+        current_folder = os.path.dirname(current_folder)
+
+    return parent_folder
 
 
-class SendActionThread(threading.Thread):
+def find_project_from_folders(folders, current_file):
+    """Find project name from open folders.
+    """
 
-    def __init__(self, target_file, is_write=False, force=False):
+    folder = find_folder_containing_file(folders, current_file)
+    return os.path.basename(folder) if folder else None
+
+
+def is_view_active(view):
+    if view:
+        active_window = sublime.active_window()
+        if active_window:
+            active_view = active_window.active_view()
+            if active_view:
+                return active_view.buffer_id() == view.buffer_id()
+    return False
+
+
+def handle_activity(view, is_write=False):
+    window = view.window()
+    if window is not None:
+        entity = view.file_name()
+        if entity:
+            timestamp = time.time()
+            last_file = LAST_HEARTBEAT['file']
+            if entity != last_file or enough_time_passed(timestamp, is_write):
+                project = window.project_data() if hasattr(window, 'project_data') else None
+                folders = window.folders()
+                append_heartbeat(entity, timestamp, is_write, view, project, folders)
+
+
+def append_heartbeat(entity, timestamp, is_write, view, project, folders):
+    global LAST_HEARTBEAT
+
+    # add this heartbeat to queue
+    heartbeat = {
+        'entity': entity,
+        'timestamp': timestamp,
+        'is_write': is_write,
+        'cursorpos': view.sel()[0].begin() if view.sel() else None,
+        'project': project,
+        'folders': folders,
+    }
+    HEARTBEATS.put_nowait(heartbeat)
+
+    # make this heartbeat the LAST_HEARTBEAT
+    LAST_HEARTBEAT = {
+        'file': entity,
+        'time': timestamp,
+        'is_write': is_write,
+    }
+
+    # process the queue of heartbeats in the future
+    seconds = 4
+    set_timeout(process_queue, seconds)
+
+
+def process_queue():
+    try:
+        heartbeat = HEARTBEATS.get_nowait()
+    except queue.Empty:
+        return
+
+    has_extra_heartbeats = False
+    extra_heartbeats = []
+    try:
+        while True:
+            extra_heartbeats.append(HEARTBEATS.get_nowait())
+            has_extra_heartbeats = True
+    except queue.Empty:
+        pass
+
+    thread = SendHeartbeatsThread(heartbeat)
+    if has_extra_heartbeats:
+        thread.add_extra_heartbeats(extra_heartbeats)
+    thread.start()
+
+
+class SendHeartbeatsThread(threading.Thread):
+    """Non-blocking thread for sending heartbeats to api.
+    """
+
+    def __init__(self, heartbeat):
         threading.Thread.__init__(self)
-        self.target_file = target_file
-        self.is_write = is_write
-        self.force = force
+
         self.debug = SETTINGS.get('debug')
         self.api_key = SETTINGS.get('api_key', '')
         self.ignore = SETTINGS.get('ignore', [])
-        self.last_action = LAST_ACTION
+
+        self.heartbeat = heartbeat
+        self.has_extra_heartbeats = False
+
+    def add_extra_heartbeats(self, extra_heartbeats):
+        self.has_extra_heartbeats = True
+        self.extra_heartbeats = extra_heartbeats
 
     def run(self):
-        if self.target_file:
-            self.timestamp = time.time()
-            if self.force or (self.is_write and not self.last_action['is_write']) or self.target_file != self.last_action['file'] or enough_time_passed(self.timestamp, self.last_action['time']):
-                self.send()
+        """Running in background thread."""
 
-    def send(self):
-        if not self.api_key:
-            print('[WakaTime] Error: missing api key.')
-            return
-        ua = 'sublime/%d sublime-wakatime/%s' % (ST_VERSION, __version__)
-        cmd = [
-            API_CLIENT,
-            '--file', self.target_file,
-            '--time', str('%f' % self.timestamp),
-            '--plugin', ua,
-            '--key', str(bytes.decode(self.api_key.encode('utf8'))),
-        ]
-        if self.is_write:
-            cmd.append('--write')
-        for pattern in self.ignore:
-            cmd.extend(['--ignore', pattern])
-        if self.debug:
-            cmd.append('--verbose')
-        if HAS_SSL:
+        self.send_heartbeats()
+
+    def build_heartbeat(self, entity=None, timestamp=None, is_write=None,
+                        cursorpos=None, project=None, folders=None):
+        """Returns a dict for passing to wakatime-cli as arguments."""
+
+        heartbeat = {
+            'entity': entity,
+            'timestamp': timestamp,
+            'is_write': is_write,
+        }
+
+        if project and project.get('name'):
+            heartbeat['alternate_project'] = project.get('name')
+        elif folders:
+            project_name = find_project_from_folders(folders, entity)
+            if project_name:
+                heartbeat['alternate_project'] = project_name
+
+        if cursorpos is not None:
+            heartbeat['cursorpos'] = '{0}'.format(cursorpos)
+
+        return heartbeat
+
+    def send_heartbeats(self):
+        if python_binary():
+            heartbeat = self.build_heartbeat(**self.heartbeat)
+            ua = 'sublime/%d sublime-wakatime/%s' % (ST_VERSION, __version__)
+            cmd = [
+                python_binary(),
+                API_CLIENT,
+                '--entity', heartbeat['entity'],
+                '--time', str('%f' % heartbeat['timestamp']),
+                '--plugin', ua,
+            ]
+            if self.api_key:
+                cmd.extend(['--key', str(bytes.decode(self.api_key.encode('utf8')))])
+            if heartbeat['is_write']:
+                cmd.append('--write')
+            if heartbeat.get('alternate_project'):
+                cmd.extend(['--alternate-project', heartbeat['alternate_project']])
+            if heartbeat.get('cursorpos') is not None:
+                cmd.extend(['--cursorpos', heartbeat['cursorpos']])
+            for pattern in self.ignore:
+                cmd.extend(['--ignore', pattern])
             if self.debug:
-                print(cmd)
-            code = wakatime.main(cmd)
-            if code != 0:
-                print('[WakaTime] Error: Response code %d from wakatime package.' % code)
-        else:
-            python = python_binary()
-            if python:
-                cmd.insert(0, python)
-                if self.debug:
-                    print(cmd)
-                if platform.system() == 'Windows':
-                    Popen(cmd, shell=False)
-                else:
-                    with open(join(expanduser('~'), '.wakatime.log'), 'a') as stderr:
-                        Popen(cmd, stderr=stderr)
+                cmd.append('--verbose')
+            if self.has_extra_heartbeats:
+                cmd.append('--extra-heartbeats')
+                stdin = PIPE
+                extra_heartbeats = [self.build_heartbeat(**x) for x in self.extra_heartbeats]
+                extra_heartbeats = json.dumps(extra_heartbeats)
             else:
-                print('[WakaTime] Error: Unable to find python binary.')
+                extra_heartbeats = None
+                stdin = None
+
+            log(DEBUG, ' '.join(obfuscate_apikey(cmd)))
+            try:
+                process = Popen(cmd, stdin=stdin, stdout=PIPE, stderr=STDOUT)
+                inp = None
+                if self.has_extra_heartbeats:
+                    inp = "{0}\n".format(extra_heartbeats)
+                    inp = inp.encode('utf-8')
+                output, err = process.communicate(input=inp)
+                output = u(output)
+                retcode = process.poll()
+                if (not retcode or retcode == 102) and not output:
+                    self.sent()
+                else:
+                    update_status_bar('Error')
+                if retcode:
+                    log(DEBUG if retcode == 102 else ERROR, 'wakatime-core exited with status: {0}'.format(retcode))
+                if output:
+                    log(ERROR, u('wakatime-core output: {0}').format(output))
+            except:
+                log(ERROR, u(sys.exc_info()[1]))
+                update_status_bar('Error')
+
+        else:
+            log(ERROR, 'Unable to find python binary.')
+            update_status_bar('Error')
+
+    def sent(self):
+        update_status_bar('OK')
+
+
+def download_python():
+    thread = DownloadPython()
+    thread.start()
+
+
+class DownloadPython(threading.Thread):
+    """Non-blocking thread for extracting embeddable Python on Windows machines.
+    """
+
+    def run(self):
+        log(INFO, 'Downloading embeddable Python...')
+
+        ver = '3.5.2'
+        arch = 'amd64' if platform.architecture()[0] == '64bit' else 'win32'
+        url = 'https://www.python.org/ftp/python/{ver}/python-{ver}-embed-{arch}.zip'.format(
+            ver=ver,
+            arch=arch,
+        )
+
+        if not os.path.exists(resources_folder()):
+            os.makedirs(resources_folder())
+
+        zip_file = os.path.join(resources_folder(), 'python.zip')
+        try:
+            urllib.urlretrieve(url, zip_file)
+        except AttributeError:
+            urllib.request.urlretrieve(url, zip_file)
+
+        log(INFO, 'Extracting Python...')
+        with contextlib.closing(ZipFile(zip_file)) as zf:
+            path = os.path.join(resources_folder(), 'python')
+            zf.extractall(path)
+
+        try:
+            os.remove(zip_file)
+        except:
+            pass
+
+        log(INFO, 'Finished extracting Python.')
 
 
 def plugin_loaded():
     global SETTINGS
     SETTINGS = sublime.load_settings(SETTINGS_FILE)
+
+    log(INFO, 'Initializing WakaTime plugin v%s' % __version__)
+    update_status_bar('Initializing')
+
+    if not python_binary():
+        log(WARNING, 'Python binary not found.')
+        if platform.system() == 'Windows':
+            set_timeout(download_python, 0)
+        else:
+            sublime.error_message("Unable to find Python binary!\nWakaTime needs Python to work correctly.\n\nGo to https://www.python.org/downloads")
+            return
+
     after_loaded()
 
 
 def after_loaded():
     if not prompt_api_key():
-        sublime.set_timeout(after_loaded, 500)
+        set_timeout(after_loaded, 0.5)
 
 
 # need to call plugin_loaded because only ST3 will auto-call it
@@ -176,10 +651,18 @@ if ST_VERSION < 3000:
 class WakatimeListener(sublime_plugin.EventListener):
 
     def on_post_save(self, view):
-        handle_action(view, is_write=True)
+        handle_activity(view, is_write=True)
 
-    def on_activated(self, view):
-        handle_action(view)
+    def on_selection_modified(self, view):
+        if is_view_active(view):
+            handle_activity(view)
 
     def on_modified(self, view):
-        handle_action(view)
+        if is_view_active(view):
+            handle_activity(view)
+
+
+class WakatimeDashboardCommand(sublime_plugin.ApplicationCommand):
+
+    def run(self):
+        webbrowser.open_new_tab('https://wakatime.com/dashboard')

WakaTime.sublime-settings

@@ -9,8 +9,15 @@
     
     // Ignore files; Files (including absolute paths) that match one of these
     // POSIX regular expressions will not be logged.
-    "ignore": ["^/tmp/", "^/etc/", "^/var/"],
+    "ignore": ["^/tmp/", "^/etc/", "^/var/", "COMMIT_EDITMSG$", "PULLREQ_EDITMSG$", "MERGE_MSG$", "TAG_EDITMSG$"],
     
     // Debug mode. Set to true for verbose logging. Defaults to false.
-    "debug": false
+    "debug": false,
+    
+    // Status bar message. Set to false to hide status bar message.
+    // Defaults to true.
+    "status_bar_message": true,
+    
+    // Status bar message format.
+    "status_bar_message_fmt": "WakaTime {status} %I:%M %p"
 }

packages/wakatime/.gitignore

@@ -1,35 +0,0 @@
-*.py[cod]
-
-# C extensions
-*.so
-
-# Packages
-*.egg
-*.egg-info
-dist
-build
-eggs
-parts
-bin
-var
-sdist
-develop-eggs
-.installed.cfg
-lib
-lib64
-
-# Installer logs
-pip-log.txt
-
-# Unit test / coverage reports
-.coverage
-.tox
-nosetests.xml
-
-# Translations
-*.mo
-
-# Mr Developer
-.mr.developer.cfg
-.project
-.pydevproject

packages/wakatime/HISTORY.rst

@@ -1,145 +0,0 @@
-
-History
--------
-
-
-2.0.0 (2014-05-25)
-++++++++++++++++++
-
-- offline time logging using sqlite3 to queue editor events
-
-
-1.0.2 (2014-05-06)
-++++++++++++++++++
-
-- ability to set project from command line argument
-
-
-1.0.1 (2014-03-05)
-++++++++++++++++++
-
-- use new domain name wakatime.com
-
-
-1.0.0 (2014-02-05)
-++++++++++++++++++
-
-- detect project name and branch name from mercurial revision control
-
-
-0.5.3 (2014-01-15)
-++++++++++++++++++
-
-- bug fix for unicode in Python3
-
-
-0.5.2 (2014-01-14)
-++++++++++++++++++
-
-- minor bug fix for Subversion on non-English systems
-
-
-0.5.1 (2013-12-13)
-++++++++++++++++++
-
-- second line in .wakatime-project file now sets branch name
-
-
-0.5.0 (2013-12-13)
-++++++++++++++++++
-
-- Convert ~/.wakatime.conf to ~/.wakatime.cfg and use configparser format
-- new [projectmap] section in cfg file for naming projects based on folders
-
-
-0.4.10 (2013-11-13)
-+++++++++++++++++++
-
-- Placing .wakatime-project file in a folder will read the project's name from that file
-
-
-0.4.9 (2013-10-27)
-++++++++++++++++++
-
-- New config for ignoring files from regular expressions
-- Parse more options from config file (verbose, logfile, ignore)
-
-
-0.4.8 (2013-10-13)
-++++++++++++++++++
-
-- Read git HEAD file to find current branch instead of running git command line
-
-
-0.4.7 (2013-09-30)
-++++++++++++++++++
-
-- Sending local olson timezone string in api request
-
-
-0.4.6 (2013-09-22)
-++++++++++++++++++
-
-- Sending total lines in file and language name to api
-
-
-0.4.5 (2013-09-07)
-++++++++++++++++++
-
-- Fixed relative import error by adding packages directory to sys path
-
-
-0.4.4 (2013-09-06)
-++++++++++++++++++
-
-- Using urllib2 again because of intermittent problems sending json with requests library
-
-
-0.4.3 (2013-09-04)
-++++++++++++++++++
-
-- Encoding json as utf-8 before making request
-
-
-0.4.2 (2013-09-04)
-++++++++++++++++++
-
-- Using requests package v1.2.3 from pypi
-
-
-0.4.1 (2013-08-25)
-++++++++++++++++++
-
-- Fix bug causing requests library to omit POST content
-
-
-0.4.0 (2013-08-15)
-++++++++++++++++++
-
-- Sending single branch instead of multiple tags
-
-
-0.3.1 (2013-08-08)
-++++++++++++++++++
-
-- Using requests module instead of urllib2 to verify SSL certs
-
-
-0.3.0 (2013-08-08)
-++++++++++++++++++
-
-- Allow importing directly from Python plugins
-
-
-0.1.1 (2013-07-07)
-++++++++++++++++++
-
-- Refactored
-- Simplified action events schema
-
-
-0.0.1 (2013-07-05)
-++++++++++++++++++
-
-- Birth
-

packages/wakatime/LICENSE

@@ -1,29 +0,0 @@
-Copyright (c) 2013 Alan Hamlett https://wakatime.com
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above copyright
-  notice, this list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright
-  notice, this list of conditions and the following disclaimer
-  in the documentation and/or other materials provided
-  with the distribution.
-
-* Neither the names of Wakatime or WakaTime, nor the names of its
-  contributors may be used to endorse or promote products derived
-  from this software without specific prior written permission.
-
-THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND
-CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
-NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER
-OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

packages/wakatime/MANIFEST.in

@@ -1,2 +0,0 @@
-include README.rst LICENSE HISTORY.rst
-recursive-include wakatime *.py

packages/wakatime/README.rst

@@ -1,20 +0,0 @@
-WakaTime
-========
-
-Fully automatic time tracking for programmers.
-
-This is the common interface for the WakaTime api. You shouldn't need to directly use this package unless you are creating a new plugin or your text editor's plugin asks you to install the wakatime-cli interface.
-
-Go to http://wakatime.com to install the plugin for your text editor.
-
-
-Installation
-------------
-
-    pip install wakatime
-
-
-Usage
------
-
-https://wakatime.com/

packages/wakatime/__about__.py

@@ -0,0 +1,9 @@
+__title__ = 'wakatime'
+__description__ = 'Common interface to the WakaTime api.'
+__url__ = 'https://github.com/wakatime/wakatime'
+__version_info__ = ('7', '0', '2')
+__version__ = '.'.join(__version_info__)
+__author__ = 'Alan Hamlett'
+__author_email__ = 'alan@wakatime.com'
+__license__ = 'BSD'
+__copyright__ = 'Copyright 2016 Alan Hamlett'

packages/wakatime/__init__.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime
+    ~~~~~~~~
+
+    Common interface to the WakaTime api.
+    http://wakatime.com
+
+    :copyright: (c) 2013 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+
+__all__ = ['main']
+
+
+from .main import execute

packages/wakatime/arguments.py

@@ -0,0 +1,236 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.arguments
+    ~~~~~~~~~~~~~~~~~~
+
+    Command-line arguments.
+
+    :copyright: (c) 2016 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+
+from __future__ import print_function
+
+
+import os
+import re
+import time
+import traceback
+from .__about__ import __version__
+from .configs import parseConfigFile
+from .constants import AUTH_ERROR
+from .packages import argparse
+
+
+class FileAction(argparse.Action):
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        try:
+            if os.path.isfile(values):
+                values = os.path.realpath(values)
+        except:  # pragma: nocover
+            pass
+        setattr(namespace, self.dest, values)
+
+
+def parseArguments():
+    """Parse command line arguments and configs from ~/.wakatime.cfg.
+    Command line arguments take precedence over config file settings.
+    Returns instances of ArgumentParser and SafeConfigParser.
+    """
+
+    # define supported command line arguments
+    parser = argparse.ArgumentParser(
+            description='Common interface for the WakaTime api.')
+    parser.add_argument('--entity', dest='entity', metavar='FILE',
+            action=FileAction,
+            help='absolute path to file for the heartbeat; can also be a '+
+                 'url, domain, or app when --entity-type is not file')
+    parser.add_argument('--file', dest='file', action=FileAction,
+            help=argparse.SUPPRESS)
+    parser.add_argument('--key', dest='key',
+            help='your wakatime api key; uses api_key from '+
+                '~/.wakatime.cfg by default')
+    parser.add_argument('--write', dest='is_write',
+            action='store_true',
+            help='when set, tells api this heartbeat was triggered from '+
+                 'writing to a file')
+    parser.add_argument('--plugin', dest='plugin',
+            help='optional text editor plugin name and version '+
+                 'for User-Agent header')
+    parser.add_argument('--time', dest='timestamp', metavar='time',
+            type=float,
+            help='optional floating-point unix epoch timestamp; '+
+                 'uses current time by default')
+    parser.add_argument('--lineno', dest='lineno',
+            help='optional line number; current line being edited')
+    parser.add_argument('--cursorpos', dest='cursorpos',
+            help='optional cursor position in the current file')
+    parser.add_argument('--entity-type', dest='entity_type',
+            help='entity type for this heartbeat. can be one of "file", '+
+                 '"domain", or "app"; defaults to file.')
+    parser.add_argument('--proxy', dest='proxy',
+                        help='optional proxy configuration. Supports HTTPS '+
+                        'and SOCKS proxies. For example: '+
+                        'https://user:pass@host:port or '+
+                        'socks5://user:pass@host:port or ' +
+                        'domain\\user:pass')
+    parser.add_argument('--project', dest='project',
+            help='optional project name')
+    parser.add_argument('--alternate-project', dest='alternate_project',
+            help='optional alternate project name; auto-discovered project '+
+                 'takes priority')
+    parser.add_argument('--alternate-language',  dest='alternate_language',
+            help=argparse.SUPPRESS)
+    parser.add_argument('--language', dest='language',
+            help='optional language name; if valid, takes priority over '+
+                 'auto-detected language')
+    parser.add_argument('--hostname', dest='hostname', help='hostname of '+
+                        'current machine.')
+    parser.add_argument('--disableoffline', dest='offline',
+            action='store_false',
+            help='disables offline time logging instead of queuing logged time')
+    parser.add_argument('--hidefilenames', dest='hidefilenames',
+            action='store_true',
+            help='obfuscate file names; will not send file names to api')
+    parser.add_argument('--exclude', dest='exclude', action='append',
+            help='filename patterns to exclude from logging; POSIX regex '+
+                 'syntax; can be used more than once')
+    parser.add_argument('--include', dest='include', action='append',
+            help='filename patterns to log; when used in combination with '+
+                 '--exclude, files matching include will still be logged; '+
+                 'POSIX regex syntax; can be used more than once')
+    parser.add_argument('--ignore', dest='ignore', action='append',
+            help=argparse.SUPPRESS)
+    parser.add_argument('--extra-heartbeats', dest='extra_heartbeats',
+            action='store_true',
+            help='reads extra heartbeats from STDIN as a JSON array until EOF')
+    parser.add_argument('--logfile', dest='logfile',
+            help='defaults to ~/.wakatime.log')
+    parser.add_argument('--apiurl', dest='api_url',
+            help='heartbeats api url; for debugging with a local server')
+    parser.add_argument('--timeout', dest='timeout', type=int,
+            help='number of seconds to wait when sending heartbeats to api; '+
+                 'defaults to 60 seconds')
+    parser.add_argument('--config', dest='config',
+            help='defaults to ~/.wakatime.cfg')
+    parser.add_argument('--verbose', dest='verbose', action='store_true',
+            help='turns on debug messages in log file')
+    parser.add_argument('--version', action='version', version=__version__)
+
+    # parse command line arguments
+    args = parser.parse_args()
+
+    # use current unix epoch timestamp by default
+    if not args.timestamp:
+        args.timestamp = time.time()
+
+    # parse ~/.wakatime.cfg file
+    configs = parseConfigFile(args.config)
+    if configs is None:
+        return args, configs
+
+    # update args from configs
+    if not args.hostname:
+        if configs.has_option('settings', 'hostname'):
+            args.hostname = configs.get('settings', 'hostname')
+    if not args.key:
+        default_key = None
+        if configs.has_option('settings', 'api_key'):
+            default_key = configs.get('settings', 'api_key')
+        elif configs.has_option('settings', 'apikey'):
+            default_key = configs.get('settings', 'apikey')
+        if default_key:
+            args.key = default_key
+        else:
+            try:
+                parser.error('Missing api key. Find your api key from wakatime.com/settings.')
+            except SystemExit:
+                raise SystemExit(AUTH_ERROR)
+
+    is_valid = not not re.match(r'^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$', args.key, re.I)
+    if not is_valid:
+        try:
+            parser.error('Invalid api key. Find your api key from wakatime.com/settings.')
+        except SystemExit:
+            raise SystemExit(AUTH_ERROR)
+
+    if not args.entity:
+        if args.file:
+            args.entity = args.file
+        else:
+            parser.error('argument --entity is required')
+
+    if not args.language and args.alternate_language:
+        args.language = args.alternate_language
+
+    if not args.exclude:
+        args.exclude = []
+    if configs.has_option('settings', 'ignore'):
+        try:
+            for pattern in configs.get('settings', 'ignore').split("\n"):
+                if pattern.strip() != '':
+                    args.exclude.append(pattern)
+        except TypeError:  # pragma: nocover
+            pass
+    if configs.has_option('settings', 'exclude'):
+        try:
+            for pattern in configs.get('settings', 'exclude').split("\n"):
+                if pattern.strip() != '':
+                    args.exclude.append(pattern)
+        except TypeError:  # pragma: nocover
+            pass
+    if not args.include:
+        args.include = []
+    if configs.has_option('settings', 'include'):
+        try:
+            for pattern in configs.get('settings', 'include').split("\n"):
+                if pattern.strip() != '':
+                    args.include.append(pattern)
+        except TypeError:  # pragma: nocover
+            pass
+    if args.hidefilenames:
+        args.hidefilenames = ['.*']
+    else:
+        args.hidefilenames = []
+        if configs.has_option('settings', 'hidefilenames'):
+            option = configs.get('settings', 'hidefilenames')
+            if option.strip().lower() == 'true':
+                args.hidefilenames = ['.*']
+            elif option.strip().lower() != 'false':
+                for pattern in option.split("\n"):
+                    if pattern.strip() != '':
+                        args.hidefilenames.append(pattern)
+    if args.offline and configs.has_option('settings', 'offline'):
+        args.offline = configs.getboolean('settings', 'offline')
+    if not args.proxy and configs.has_option('settings', 'proxy'):
+        args.proxy = configs.get('settings', 'proxy')
+    if args.proxy:
+        pattern = r'^((https?|socks5)://)?([^:@]+(:([^:@])+)?@)?[^:]+(:\d+)?$'
+        if '\\' in args.proxy:
+            pattern = r'^.*\\.+$'
+        is_valid = not not re.match(pattern, args.proxy, re.I)
+        if not is_valid:
+            parser.error('Invalid proxy. Must be in format ' +
+                            'https://user:pass@host:port or ' +
+                            'socks5://user:pass@host:port or ' +
+                            'domain\\user:pass.')
+    if not args.verbose and configs.has_option('settings', 'verbose'):
+        args.verbose = configs.getboolean('settings', 'verbose')
+    if not args.verbose and configs.has_option('settings', 'debug'):
+        args.verbose = configs.getboolean('settings', 'debug')
+    if not args.logfile and configs.has_option('settings', 'logfile'):
+        args.logfile = configs.get('settings', 'logfile')
+    if not args.logfile and os.environ.get('WAKATIME_HOME'):
+        home = os.environ.get('WAKATIME_HOME')
+        args.logfile = os.path.join(os.path.expanduser(home), '.wakatime.log')
+    if not args.api_url and configs.has_option('settings', 'api_url'):
+        args.api_url = configs.get('settings', 'api_url')
+    if not args.timeout and configs.has_option('settings', 'timeout'):
+        try:
+            args.timeout = int(configs.get('settings', 'timeout'))
+        except ValueError:
+            print(traceback.format_exc())
+
+    return args, configs

packages/wakatime/cli.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.cli
+    ~~~~~~~~~~~~
+
+    Command-line entry point.
+
+    :copyright: (c) 2013 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import os
+import sys
+
+
+# get path to local wakatime package
+package_folder = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+# add local wakatime package to sys.path
+sys.path.insert(0, package_folder)
+
+# import local wakatime package
+try:
+    import wakatime
+except (TypeError, ImportError):
+    # on Windows, non-ASCII characters in import path can be fixed using
+    # the script path from sys.argv[0].
+    # More info at https://github.com/wakatime/wakatime/issues/32
+    package_folder = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
+    sys.path.insert(0, package_folder)
+    import wakatime
+
+
+if __name__ == '__main__':
+    sys.exit(wakatime.execute(sys.argv[1:]))

packages/wakatime/compat.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.compat
+    ~~~~~~~~~~~~~~~
+
+    For working with Python2 and Python3.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import codecs
+import sys
+
+
+is_py2 = (sys.version_info[0] == 2)
+is_py3 = (sys.version_info[0] == 3)
+
+
+if is_py2:  # pragma: nocover
+
+    def u(text):
+        if text is None:
+            return None
+        try:
+            return text.decode('utf-8')
+        except:
+            try:
+                return text.decode(sys.getdefaultencoding())
+            except:
+                try:
+                    return unicode(text)
+                except:
+                    return text.decode('utf-8', 'replace')
+    open = codecs.open
+    basestring = basestring
+
+
+elif is_py3:  # pragma: nocover
+
+    def u(text):
+        if text is None:
+            return None
+        if isinstance(text, bytes):
+            try:
+                return text.decode('utf-8')
+            except:
+                try:
+                    return text.decode(sys.getdefaultencoding())
+                except:
+                    pass
+        try:
+            return str(text)
+        except:
+            return text.decode('utf-8', 'replace')
+    open = open
+    basestring = (str, bytes)
+
+
+try:
+    from importlib import import_module
+except ImportError:  # pragma: nocover
+    def _resolve_name(name, package, level):
+        """Return the absolute name of the module to be imported."""
+        if not hasattr(package, 'rindex'):
+            raise ValueError("'package' not set to a string")
+        dot = len(package)
+        for x in xrange(level, 1, -1):
+            try:
+                dot = package.rindex('.', 0, dot)
+            except ValueError:
+                raise ValueError("attempted relative import beyond top-level "
+                                 "package")
+        return "%s.%s" % (package[:dot], name)
+
+    def import_module(name, package=None):
+        """Import a module.
+        The 'package' argument is required when performing a relative import.
+        It specifies the package to use as the anchor point from which to
+        resolve the relative import to an absolute import.
+        """
+        if name.startswith('.'):
+            if not package:
+                raise TypeError("relative imports require the 'package' "
+                                + "argument")
+            level = 0
+            for character in name:
+                if character != '.':
+                    break
+                level += 1
+            name = _resolve_name(name[level:], package, level)
+        __import__(name)
+        return sys.modules[name]

packages/wakatime/configs.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.configs
+    ~~~~~~~~~~~~~~~~
+
+    Config file parser.
+
+    :copyright: (c) 2016 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+
+from __future__ import print_function
+
+import os
+import sys
+import traceback
+
+from .compat import u, open
+from .constants import CONFIG_FILE_PARSE_ERROR
+
+
+try:
+    import ConfigParser as configparser
+except ImportError:  # pragma: nocover
+    import configparser
+
+
+def parseConfigFile(configFile=None):
+    """Returns a configparser.SafeConfigParser instance with configs
+    read from the config file. Default location of the config file is
+    at ~/.wakatime.cfg.
+    """
+
+    # get config file location from ENV
+    home = os.environ.get('WAKATIME_HOME')
+    if not configFile and home:
+        configFile = os.path.join(os.path.expanduser(home), '.wakatime.cfg')
+
+    # use default config file location
+    if not configFile:
+        configFile = os.path.join(os.path.expanduser('~'), '.wakatime.cfg')
+
+    configs = configparser.SafeConfigParser()
+    try:
+        with open(configFile, 'r', encoding='utf-8') as fh:
+            try:
+                configs.readfp(fh)
+            except configparser.Error:
+                print(traceback.format_exc())
+                return None
+    except IOError:
+        sys.stderr.write(u("Error: Could not read from config file {0}\n").format(u(configFile)))
+        raise SystemExit(CONFIG_FILE_PARSE_ERROR)
+    return configs

packages/wakatime/constants.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.constants
+    ~~~~~~~~~~~~~~~~~~
+
+    Constant variable definitions.
+
+    :copyright: (c) 2016 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+""" Success
+Exit code used when a heartbeat was sent successfully.
+"""
+SUCCESS = 0
+
+""" Api Error
+Exit code used when the WakaTime API returned an error.
+"""
+API_ERROR = 102
+
+""" Config File Parse Error
+Exit code used when the ~/.wakatime.cfg config file could not be parsed.
+"""
+CONFIG_FILE_PARSE_ERROR = 103
+
+""" Auth Error
+Exit code used when our api key is invalid.
+"""
+AUTH_ERROR = 104
+
+""" Unknown Error
+Exit code used when there was an unhandled exception.
+"""
+UNKNOWN_ERROR = 105
+
+""" Malformed Heartbeat Error
+Exit code used when the JSON input from `--extra-heartbeats` is malformed.
+"""
+MALFORMED_HEARTBEAT_ERROR = 106

packages/wakatime/dependencies/__init__.py

@@ -0,0 +1,129 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from a source code file.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import logging
+import re
+import sys
+
+from ..compat import u, open, import_module
+from ..exceptions import NotYetImplemented
+
+
+log = logging.getLogger('WakaTime')
+
+
+class TokenParser(object):
+    """The base class for all dependency parsers. To add support for your
+    language, inherit from this class and implement the :meth:`parse` method
+    to return a list of dependency strings.
+    """
+    exclude = []
+
+    def __init__(self, source_file, lexer=None):
+        self._tokens = None
+        self.dependencies = []
+        self.source_file = source_file
+        self.lexer = lexer
+        self.exclude = [re.compile(x, re.IGNORECASE) for x in self.exclude]
+
+    @property
+    def tokens(self):
+        if self._tokens is None:
+            self._tokens = self._extract_tokens()
+        return self._tokens
+
+    def parse(self, tokens=[]):
+        """ Should return a list of dependencies.
+        """
+        raise NotYetImplemented()
+
+    def append(self, dep, truncate=False, separator=None, truncate_to=None,
+               strip_whitespace=True):
+        self._save_dependency(
+            dep,
+            truncate=truncate,
+            truncate_to=truncate_to,
+            separator=separator,
+            strip_whitespace=strip_whitespace,
+        )
+
+    def partial(self, token):
+        return u(token).split('.')[-1]
+
+    def _extract_tokens(self):
+        if self.lexer:
+            try:
+                with open(self.source_file, 'r', encoding='utf-8') as fh:
+                    return self.lexer.get_tokens_unprocessed(fh.read(512000))
+            except:
+                pass
+            try:
+                with open(self.source_file, 'r', encoding=sys.getfilesystemencoding()) as fh:
+                    return self.lexer.get_tokens_unprocessed(fh.read(512000))  # pragma: nocover
+            except:
+                pass
+        return []
+
+    def _save_dependency(self, dep, truncate=False, separator=None,
+                         truncate_to=None, strip_whitespace=True):
+        if truncate:
+            if separator is None:
+                separator = u('.')
+            separator = u(separator)
+            dep = dep.split(separator)
+            if truncate_to is None or truncate_to < 1:
+                truncate_to = 1
+            if truncate_to > len(dep):
+                truncate_to = len(dep)
+            dep = dep[0] if len(dep) == 1 else separator.join(dep[:truncate_to])
+        if strip_whitespace:
+            dep = dep.strip()
+        if dep and (not separator or not dep.startswith(separator)):
+            should_exclude = False
+            for compiled in self.exclude:
+                if compiled.search(dep):
+                    should_exclude = True
+                    break
+            if not should_exclude:
+                self.dependencies.append(dep)
+
+
+class DependencyParser(object):
+    source_file = None
+    lexer = None
+    parser = None
+
+    def __init__(self, source_file, lexer):
+        self.source_file = source_file
+        self.lexer = lexer
+
+        if self.lexer:
+            module_name = self.lexer.__module__.rsplit('.', 1)[-1]
+            class_name = self.lexer.__class__.__name__.replace('Lexer', 'Parser', 1)
+        else:
+            module_name = 'unknown'
+            class_name = 'UnknownParser'
+
+        try:
+            module = import_module('.%s' % module_name, package=__package__)
+            try:
+                self.parser = getattr(module, class_name)
+            except AttributeError:
+                log.debug('Parsing dependencies not supported for {0}.{1}'.format(module_name, class_name))
+        except ImportError:
+            log.debug('Parsing dependencies not supported for {0}.{1}'.format(module_name, class_name))
+
+    def parse(self):
+        if self.parser:
+            plugin = self.parser(self.source_file, lexer=self.lexer)
+            dependencies = plugin.parse()
+            return list(set(dependencies))
+        return []

packages/wakatime/dependencies/c_cpp.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.languages.c_cpp
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from C++ code.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+
+
+class CParser(TokenParser):
+    exclude = [
+        r'^stdio\.h$',
+        r'^stdlib\.h$',
+        r'^string\.h$',
+        r'^time\.h$',
+    ]
+    state = None
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Preproc' or self.partial(token) == 'PreprocFile':
+            self._process_preproc(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_preproc(self, token, content):
+        if self.state == 'include':
+            if content != '\n' and content != '#':
+                content = content.strip().strip('"').strip('<').strip('>').strip()
+                self.append(content, truncate=True, separator='/')
+            self.state = None
+        elif content.strip().startswith('include'):
+            self.state = 'include'
+        else:
+            self.state = None
+
+    def _process_other(self, token, content):
+        pass
+
+
+class CppParser(CParser):
+    pass

packages/wakatime/dependencies/data.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.languages.data
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from data files.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import os
+
+from . import TokenParser
+from ..compat import u
+
+
+FILES = {
+    'bower.json': {'exact': True, 'dependency': 'bower'},
+    'component.json': {'exact': True, 'dependency': 'bower'},
+    'package.json': {'exact': True, 'dependency': 'npm'},
+}
+
+
+class JsonParser(TokenParser):
+    state = None
+    level = 0
+
+    def parse(self):
+        self._process_file_name(os.path.basename(self.source_file))
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_file_name(self, file_name):
+        for key, value in FILES.items():
+            found = (key == file_name) if value.get('exact') else (key.lower() in file_name.lower())
+            if found:
+                self.append(value['dependency'])
+
+    def _process_token(self, token, content):
+        if u(token) == 'Token.Name.Tag':
+            self._process_tag(token, content)
+        elif u(token) == 'Token.Literal.String.Single' or u(token) == 'Token.Literal.String.Double':
+            self._process_literal_string(token, content)
+        elif u(token) == 'Token.Punctuation':
+            self._process_punctuation(token, content)
+
+    def _process_tag(self, token, content):
+        if content.strip('"').strip("'") == 'dependencies' or content.strip('"').strip("'") == 'devDependencies':
+            self.state = 'dependencies'
+        elif self.state == 'dependencies' and self.level == 2:
+            self.append(content.strip('"').strip("'"))
+
+    def _process_literal_string(self, token, content):
+        pass
+
+    def _process_punctuation(self, token, content):
+        if content == '{':
+            self.level += 1
+        elif content == '}':
+            self.level -= 1
+            if self.state is not None and self.level <= 1:
+                self.state = None

packages/wakatime/dependencies/dotnet.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.languages.dotnet
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from .NET code.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+from ..compat import u
+
+
+class CSharpParser(TokenParser):
+    exclude = [
+        r'^system$',
+        r'^microsoft$',
+    ]
+    state = None
+    buffer = u('')
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Keyword':
+            self._process_keyword(token, content)
+        if self.partial(token) == 'Namespace' or self.partial(token) == 'Name':
+            self._process_namespace(token, content)
+        elif self.partial(token) == 'Punctuation':
+            self._process_punctuation(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_keyword(self, token, content):
+        if content == 'using':
+            self.state = 'import'
+            self.buffer = u('')
+
+    def _process_namespace(self, token, content):
+        if self.state == 'import':
+            if u(content) != u('import') and u(content) != u('package') and u(content) != u('namespace') and u(content) != u('static'):
+                if u(content) == u(';'):  # pragma: nocover
+                    self._process_punctuation(token, content)
+                else:
+                    self.buffer += u(content)
+
+    def _process_punctuation(self, token, content):
+        if self.state == 'import':
+            if u(content) == u(';'):
+                self.append(self.buffer, truncate=True)
+                self.buffer = u('')
+                self.state = None
+            elif u(content) == u('='):
+                self.buffer = u('')
+            else:
+                self.buffer += u(content)
+
+    def _process_other(self, token, content):
+        pass

packages/wakatime/dependencies/go.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.languages.go
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Go code.
+
+    :copyright: (c) 2016 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+
+
+class GoParser(TokenParser):
+    state = None
+    parens = 0
+    aliases = 0
+    exclude = [
+        r'^"fmt"$',
+    ]
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        elif self.partial(token) == 'Punctuation':
+            self._process_punctuation(token, content)
+        elif self.partial(token) == 'String':
+            self._process_string(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        elif self.partial(token) == 'Other':
+            self._process_other(token, content)
+        else:
+            self._process_misc(token, content)
+
+    def _process_namespace(self, token, content):
+        self.state = content
+        self.parens = 0
+        self.aliases = 0
+
+    def _process_string(self, token, content):
+        if self.state == 'import':
+            self.append(content, truncate=False)
+
+    def _process_punctuation(self, token, content):
+        if content == '(':
+            self.parens += 1
+        elif content == ')':
+            self.parens -= 1
+        elif content == '.':
+            self.aliases += 1
+        else:
+            self.state = None
+
+    def _process_text(self, token, content):
+        if self.state == 'import':
+            if content == "\n" and self.parens <= 0:
+                self.state = None
+                self.parens = 0
+                self.aliases = 0
+        else:
+            self.state = None
+
+    def _process_other(self, token, content):
+        if self.state == 'import':
+            self.aliases += 1
+        else:
+            self.state = None
+
+    def _process_misc(self, token, content):
+        self.state = None

packages/wakatime/dependencies/jvm.py

@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.languages.java
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Java code.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+from ..compat import u
+
+
+class JavaParser(TokenParser):
+    exclude = [
+        r'^java\.',
+        r'^javax\.',
+        r'^import$',
+        r'^package$',
+        r'^namespace$',
+        r'^static$',
+    ]
+    state = None
+    buffer = u('')
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        if self.partial(token) == 'Name':
+            self._process_name(token, content)
+        elif self.partial(token) == 'Attribute':
+            self._process_attribute(token, content)
+        elif self.partial(token) == 'Operator':
+            self._process_operator(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_namespace(self, token, content):
+        if u(content) == u('import'):
+            self.state = 'import'
+
+        elif self.state == 'import':
+            keywords = [
+                u('package'),
+                u('namespace'),
+                u('static'),
+            ]
+            if u(content) in keywords:
+                return
+            self.buffer = u('{0}{1}').format(self.buffer, u(content))
+
+        elif self.state == 'import-finished':
+            content = content.split(u('.'))
+
+            if len(content) == 1:
+                self.append(content[0])
+
+            elif len(content) > 1:
+                if len(content[0]) == 3:
+                    content = content[1:]
+                if content[-1] == u('*'):
+                    content = content[:len(content) - 1]
+
+                if len(content) == 1:
+                    self.append(content[0])
+                elif len(content) > 1:
+                    self.append(u('.').join(content[:2]))
+
+            self.state = None
+
+    def _process_name(self, token, content):
+        if self.state == 'import':
+            self.buffer = u('{0}{1}').format(self.buffer, u(content))
+
+    def _process_attribute(self, token, content):
+        if self.state == 'import':
+            self.buffer = u('{0}{1}').format(self.buffer, u(content))
+
+    def _process_operator(self, token, content):
+        if u(content) == u(';'):
+            self.state = 'import-finished'
+            self._process_namespace(token, self.buffer)
+            self.state = None
+            self.buffer = u('')
+        elif self.state == 'import':
+            self.buffer = u('{0}{1}').format(self.buffer, u(content))
+
+    def _process_other(self, token, content):
+        pass

packages/wakatime/dependencies/php.py

@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.languages.php
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from PHP code.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+from ..compat import u
+
+
+class PhpParser(TokenParser):
+    state = None
+    parens = 0
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Keyword':
+            self._process_keyword(token, content)
+        elif u(token) == 'Token.Literal.String.Single' or u(token) == 'Token.Literal.String.Double':
+            self._process_literal_string(token, content)
+        elif u(token) == 'Token.Name.Other':
+            self._process_name(token, content)
+        elif u(token) == 'Token.Name.Function':
+            self._process_function(token, content)
+        elif self.partial(token) == 'Punctuation':
+            self._process_punctuation(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_name(self, token, content):
+        if self.state == 'use':
+            self.append(content, truncate=True, separator=u("\\"))
+
+    def _process_function(self, token, content):
+        if self.state == 'use function':
+            self.append(content, truncate=True, separator=u("\\"))
+            self.state = 'use'
+
+    def _process_keyword(self, token, content):
+        if content == 'include' or content == 'include_once' or content == 'require' or content == 'require_once':
+            self.state = 'include'
+        elif content == 'use':
+            self.state = 'use'
+        elif content == 'as':
+            self.state = 'as'
+        elif self.state == 'use' and content == 'function':
+            self.state = 'use function'
+        else:
+            self.state = None
+
+    def _process_literal_string(self, token, content):
+        if self.state == 'include':
+            if content != '"' and content != "'":
+                content = content.strip()
+                if u(token) == 'Token.Literal.String.Double':
+                    content = u("'{0}'").format(content)
+                self.append(content)
+                self.state = None
+
+    def _process_punctuation(self, token, content):
+        if content == '(':
+            self.parens += 1
+        elif content == ')':
+            self.parens -= 1
+        elif (self.state == 'use' or self.state == 'as') and content == ',':
+            self.state = 'use'
+        else:
+            self.state = None
+
+    def _process_text(self, token, content):
+        pass
+
+    def _process_other(self, token, content):
+        self.state = None

packages/wakatime/dependencies/python.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.languages.python
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Python code.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+
+
+class PythonParser(TokenParser):
+    state = None
+    parens = 0
+    nonpackage = False
+    exclude = [
+        r'^os$',
+        r'^sys$',
+        r'^sys\.',
+        r'^__future__$',
+    ]
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        elif self.partial(token) == 'Operator':
+            self._process_operator(token, content)
+        elif self.partial(token) == 'Punctuation':
+            self._process_punctuation(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_namespace(self, token, content):
+        if self.state is None:
+            self.state = content
+        else:
+            if content == 'as':
+                self.nonpackage = True
+            else:
+                self._process_import(token, content)
+
+    def _process_operator(self, token, content):
+        pass
+
+    def _process_punctuation(self, token, content):
+        if content == '(':
+            self.parens += 1
+        elif content == ')':
+            self.parens -= 1
+        self.nonpackage = False
+
+    def _process_text(self, token, content):
+        if self.state is not None:
+            if content == "\n" and self.parens == 0:
+                self.state = None
+                self.nonpackage = False
+
+    def _process_other(self, token, content):
+        pass
+
+    def _process_import(self, token, content):
+        if not self.nonpackage:
+            if self.state == 'from':
+                self.append(content, truncate=True, truncate_to=1)
+                self.state = 'from-2'
+            elif self.state == 'import':
+                self.append(content, truncate=True, truncate_to=1)
+                self.state = 'import-2'
+            elif self.state == 'import-2':
+                self.append(content, truncate=True, truncate_to=1)
+            else:
+                self.state = None
+        self.nonpackage = False

packages/wakatime/dependencies/templates.py

@@ -0,0 +1,203 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.languages.templates
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Templates.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+from ..compat import u
+
+
+""" If these keywords are found in the source file, treat them as a dependency.
+Must be lower-case strings.
+"""
+KEYWORDS = [
+    '_',
+    '$',
+    'angular',
+    'assert', # probably mocha
+    'backbone',
+    'batman',
+    'c3',
+    'can',
+    'casper',
+    'chai',
+    'chaplin',
+    'd3',
+    'define', # probably require
+    'describe', # mocha or jasmine
+    'eco',
+    'ember',
+    'espresso',
+    'expect', # probably jasmine
+    'exports', # probably npm
+    'express',
+    'gulp',
+    'handlebars',
+    'highcharts',
+    'jasmine',
+    'jquery',
+    'jstz',
+    'ko', # probably knockout
+    'm', # probably mithril
+    'marionette',
+    'meteor',
+    'moment',
+    'monitorio',
+    'mustache',
+    'phantom',
+    'pickadate',
+    'pikaday',
+    'qunit',
+    'react',
+    'reactive',
+    'require', # probably the commonjs spec
+    'ripple',
+    'rivets',
+    'socketio',
+    'spine',
+    'thorax',
+    'underscore',
+    'vue',
+    'way',
+    'zombie',
+]
+
+
+class HtmlDjangoParser(TokenParser):
+    tags = []
+    opening_tag = False
+    getting_attrs = False
+    current_attr = None
+    current_attr_value = None
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if u(token) == 'Token.Punctuation':
+            self._process_punctuation(token, content)
+        elif u(token) == 'Token.Name.Tag':
+            self._process_tag(token, content)
+        elif u(token) == 'Token.Literal.String':
+            self._process_string(token, content)
+        elif u(token) == 'Token.Name.Attribute':
+            self._process_attribute(token, content)
+
+    @property
+    def current_tag(self):
+        return None if len(self.tags) == 0 else self.tags[0]
+
+    def _process_punctuation(self, token, content):
+        if content.startswith('</') or content.startswith('/'):
+            try:
+                self.tags.pop(0)
+            except IndexError:
+                # ignore errors from malformed markup
+                pass
+            self.opening_tag = False
+            self.getting_attrs = False
+        elif content.startswith('<'):
+            self.opening_tag = True
+        elif content.startswith('>'):
+            self.opening_tag = False
+            self.getting_attrs = False
+
+    def _process_tag(self, token, content):
+        if self.opening_tag:
+            self.tags.insert(0, content.replace('<', '', 1).strip().lower())
+            self.getting_attrs = True
+        self.current_attr = None
+
+    def _process_attribute(self, token, content):
+        if self.getting_attrs:
+            self.current_attr = content.lower().strip('=')
+        self.current_attr_value = None
+
+    def _process_string(self, token, content):
+        if self.getting_attrs and self.current_attr is not None:
+            if content.endswith('"') or content.endswith("'"):
+                if self.current_attr_value is not None:
+                    self.current_attr_value += content
+                    if self.current_tag == 'script' and self.current_attr == 'src':
+                        self.append(self.current_attr_value)
+                    self.current_attr = None
+                    self.current_attr_value = None
+                else:
+                    if len(content) == 1:
+                        self.current_attr_value = content
+                    else:
+                        if self.current_tag == 'script' and self.current_attr == 'src':
+                            self.append(content)
+                        self.current_attr = None
+                        self.current_attr_value = None
+            elif content.startswith('"') or content.startswith("'"):
+                if self.current_attr_value is None:
+                    self.current_attr_value = content
+
+
+class VelocityHtmlParser(HtmlDjangoParser):
+    pass
+
+
+class MyghtyHtmlParser(HtmlDjangoParser):
+    pass
+
+
+class MasonParser(HtmlDjangoParser):
+    pass
+
+
+class MakoHtmlParser(HtmlDjangoParser):
+    pass
+
+
+class CheetahHtmlParser(HtmlDjangoParser):
+    pass
+
+
+class HtmlGenshiParser(HtmlDjangoParser):
+    pass
+
+
+class RhtmlParser(HtmlDjangoParser):
+    pass
+
+
+class HtmlPhpParser(HtmlDjangoParser):
+    pass
+
+
+class HtmlSmartyParser(HtmlDjangoParser):
+    pass
+
+
+class EvoqueHtmlParser(HtmlDjangoParser):
+    pass
+
+
+class ColdfusionHtmlParser(HtmlDjangoParser):
+    pass
+
+
+class LassoHtmlParser(HtmlDjangoParser):
+    pass
+
+
+class HandlebarsHtmlParser(HtmlDjangoParser):
+    pass
+
+
+class YamlJinjaParser(HtmlDjangoParser):
+    pass
+
+
+class TwigHtmlParser(HtmlDjangoParser):
+    pass

packages/wakatime/dependencies/unknown.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.languages.unknown
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from files of unknown language.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import os
+
+from . import TokenParser
+
+
+FILES = {
+    'bower': {'exact': False, 'dependency': 'bower'},
+    'grunt': {'exact': False, 'dependency': 'grunt'},
+}
+
+
+class UnknownParser(TokenParser):
+
+    def parse(self):
+        self._process_file_name(os.path.basename(self.source_file))
+        return self.dependencies
+
+    def _process_file_name(self, file_name):
+        for key, value in FILES.items():
+            found = (key == file_name) if value.get('exact') else (key.lower() in file_name.lower())
+            if found:
+                self.append(value['dependency'])

packages/wakatime/exceptions.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.exceptions
+    ~~~~~~~~~~~~~~~~~~~
+
+    Custom exceptions.
+
+    :copyright: (c) 2015 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+
+class NotYetImplemented(Exception):
+    """This method needs to be implemented."""

packages/wakatime/languages/default.json

@@ -0,0 +1,81 @@
+{
+  "actionscript": "ActionScript", 
+  "apacheconf": "ApacheConf", 
+  "applescript": "AppleScript", 
+  "asp": "ASP", 
+  "assembly": "Assembly", 
+  "awk": "Awk", 
+  "bash": "Bash", 
+  "basic": "Basic", 
+  "brightscript": "BrightScript", 
+  "c": "C", 
+  "c#": "C#", 
+  "c++": "C++", 
+  "clojure": "Clojure", 
+  "cocoa": "Cocoa", 
+  "coffeescript": "CoffeeScript", 
+  "coldfusion": "ColdFusion", 
+  "common lisp": "Common Lisp", 
+  "cshtml": "CSHTML", 
+  "css": "CSS", 
+  "dart": "Dart", 
+  "delphi": "Delphi", 
+  "elixir": "Elixir", 
+  "elm": "Elm", 
+  "emacs lisp": "Emacs Lisp", 
+  "erlang": "Erlang", 
+  "f#": "F#", 
+  "fortran": "Fortran", 
+  "go": "Go", 
+  "gous": "Gosu", 
+  "groovy": "Groovy", 
+  "haml": "Haml", 
+  "haskell": "Haskell", 
+  "haxe": "HaXe", 
+  "html": "HTML", 
+  "ini": "INI", 
+  "jade": "Jade", 
+  "java": "Java", 
+  "javascript": "JavaScript", 
+  "json": "JSON", 
+  "jsx": "JSX", 
+  "kotlin": "Kotlin", 
+  "less": "LESS", 
+  "lua": "Lua", 
+  "markdown": "Markdown", 
+  "matlab": "Matlab", 
+  "mustache": "Mustache", 
+  "objective-c": "Objective-C", 
+  "objective-c++": "Objective-C++", 
+  "objective-j": "Objective-J", 
+  "ocaml": "OCaml", 
+  "perl": "Perl", 
+  "php": "PHP", 
+  "powershell": "PowerShell", 
+  "prolog": "Prolog", 
+  "puppet": "Puppet", 
+  "python": "Python", 
+  "r": "R", 
+  "restructuredtext": "reStructuredText", 
+  "ruby": "Ruby", 
+  "rust": "Rust", 
+  "sass": "Sass", 
+  "scala": "Scala", 
+  "scheme": "Scheme", 
+  "scss": "SCSS", 
+  "shell": "Shell", 
+  "slim": "Slim", 
+  "smalltalk": "Smalltalk", 
+  "sql": "SQL", 
+  "swift": "Swift", 
+  "text": "Text", 
+  "turing": "Turing", 
+  "twig": "Twig", 
+  "typescript": "TypeScript", 
+  "typoscript": "TypoScript", 
+  "vb.net": "VB.net", 
+  "viml": "VimL", 
+  "xaml": "XAML", 
+  "xml": "XML", 
+  "yaml": "YAML"
+}

packages/wakatime/languages/vim.json

@@ -0,0 +1,531 @@
+{
+  "a2ps": null,
+  "a65": "Assembly",
+  "aap": null,
+  "abap": null,
+  "abaqus": null,
+  "abc": null,
+  "abel": null,
+  "acedb": null,
+  "ada": null,
+  "aflex": null,
+  "ahdl": null,
+  "alsaconf": null,
+  "amiga": null,
+  "aml": null,
+  "ampl": null,
+  "ant": null,
+  "antlr": null,
+  "apache": null,
+  "apachestyle": null,
+  "arch": null,
+  "art": null,
+  "asm": "Assembly",
+  "asm68k": "Assembly",
+  "asmh8300": "Assembly",
+  "asn": null,
+  "aspperl": null,
+  "aspvbs": null,
+  "asterisk": null,
+  "asteriskvm": null,
+  "atlas": null,
+  "autohotkey": null,
+  "autoit": null,
+  "automake": null,
+  "ave": null,
+  "awk": null,
+  "ayacc": null,
+  "b": null,
+  "baan": null,
+  "basic": "Basic",
+  "bc": null,
+  "bdf": null,
+  "bib": null,
+  "bindzone": null,
+  "blank": null,
+  "bst": null,
+  "btm": null,
+  "bzr": null,
+  "c": "C",
+  "cabal": null,
+  "calendar": null,
+  "catalog": null,
+  "cdl": null,
+  "cdrdaoconf": null,
+  "cdrtoc": null,
+  "cf": null,
+  "cfg": null,
+  "ch": null,
+  "chaiscript": null,
+  "change": null,
+  "changelog": null,
+  "chaskell": null,
+  "cheetah": null,
+  "chill": null,
+  "chordpro": null,
+  "cl": null,
+  "clean": null,
+  "clipper": null,
+  "cmake": null,
+  "cmusrc": null,
+  "cobol": null,
+  "coco": null,
+  "conaryrecipe": null,
+  "conf": null,
+  "config": null,
+  "context": null,
+  "cpp": "C++",
+  "crm": null,
+  "crontab": "Crontab",
+  "cs": "C#",
+  "csc": null,
+  "csh": null,
+  "csp": null,
+  "css": null,
+  "cterm": null,
+  "ctrlh": null,
+  "cucumber": null,
+  "cuda": null,
+  "cupl": null,
+  "cuplsim": null,
+  "cvs": null,
+  "cvsrc": null,
+  "cweb": null,
+  "cynlib": null,
+  "cynpp": null,
+  "d": null,
+  "datascript": null,
+  "dcd": null,
+  "dcl": null,
+  "debchangelog": null,
+  "debcontrol": null,
+  "debsources": null,
+  "def": null,
+  "denyhosts": null,
+  "desc": null,
+  "desktop": null,
+  "dictconf": null,
+  "dictdconf": null,
+  "diff": null,
+  "dircolors": null,
+  "diva": null,
+  "django": null,
+  "dns": null,
+  "docbk": null,
+  "docbksgml": null,
+  "docbkxml": null,
+  "dosbatch": null,
+  "dosini": null,
+  "dot": null,
+  "doxygen": null,
+  "dracula": null,
+  "dsl": null,
+  "dtd": null,
+  "dtml": null,
+  "dtrace": null,
+  "dylan": null,
+  "dylanintr": null,
+  "dylanlid": null,
+  "ecd": null,
+  "edif": null,
+  "eiffel": null,
+  "elf": null,
+  "elinks": null,
+  "elmfilt": null,
+  "erlang": null,
+  "eruby": null,
+  "esmtprc": null,
+  "esqlc": null,
+  "esterel": null,
+  "eterm": null,
+  "eviews": null,
+  "exim": null,
+  "expect": null,
+  "exports": null,
+  "fan": null,
+  "fasm": null,
+  "fdcc": null,
+  "fetchmail": null,
+  "fgl": null,
+  "flexwiki": null,
+  "focexec": null,
+  "form": null,
+  "forth": null,
+  "fortran": null,
+  "foxpro": null,
+  "framescript": null,
+  "freebasic": null,
+  "fstab": null,
+  "fvwm": null,
+  "fvwm2m4": null,
+  "gdb": null,
+  "gdmo": null,
+  "gedcom": null,
+  "git": null,
+  "gitcommit": null,
+  "gitconfig": null,
+  "gitrebase": null,
+  "gitsendemail": null,
+  "gkrellmrc": null,
+  "gnuplot": null,
+  "gp": null,
+  "gpg": null,
+  "grads": null,
+  "gretl": null,
+  "groff": null,
+  "groovy": null,
+  "group": null,
+  "grub": null,
+  "gsp": null,
+  "gtkrc": null,
+  "haml": "Haml",
+  "hamster": null,
+  "haskell": "Haskell",
+  "haste": null,
+  "hastepreproc": null,
+  "hb": null,
+  "help": null,
+  "hercules": null,
+  "hex": null,
+  "hog": null,
+  "hostconf": null,
+  "html": "HTML",
+  "htmlcheetah": "HTML",
+  "htmldjango": "HTML",
+  "htmlm4": "HTML",
+  "htmlos": null,
+  "ia64": null,
+  "ibasic": null,
+  "icemenu": null,
+  "icon": null,
+  "idl": null,
+  "idlang": null,
+  "indent": null,
+  "inform": null,
+  "initex": null,
+  "initng": null,
+  "inittab": null,
+  "ipfilter": null,
+  "ishd": null,
+  "iss": null,
+  "ist": null,
+  "jal": null,
+  "jam": null,
+  "jargon": null,
+  "java": "Java",
+  "javacc": null,
+  "javascript": "JavaScript",
+  "jess": null,
+  "jgraph": null,
+  "jproperties": null,
+  "jsp": null,
+  "kconfig": null,
+  "kix": null,
+  "kscript": null,
+  "kwt": null,
+  "lace": null,
+  "latte": null,
+  "ld": null,
+  "ldapconf": null,
+  "ldif": null,
+  "lex": null,
+  "lftp": null,
+  "lhaskell": "Haskell",
+  "libao": null,
+  "lifelines": null,
+  "lilo": null,
+  "limits": null,
+  "liquid": null,
+  "lisp": null,
+  "lite": null,
+  "litestep": null,
+  "loginaccess": null,
+  "logindefs": null,
+  "logtalk": null,
+  "lotos": null,
+  "lout": null,
+  "lpc": null,
+  "lprolog": null,
+  "lscript": null,
+  "lsl": null,
+  "lss": null,
+  "lua": null,
+  "lynx": null,
+  "m4": null,
+  "mail": null,
+  "mailaliases": null,
+  "mailcap": null,
+  "make": null,
+  "man": null,
+  "manconf": null,
+  "manual": null,
+  "maple": null,
+  "markdown": "Markdown",
+  "masm": null,
+  "mason": null,
+  "master": null,
+  "matlab": null,
+  "maxima": null,
+  "mel": null,
+  "messages": null,
+  "mf": null,
+  "mgl": null,
+  "mgp": null,
+  "mib": null,
+  "mma": null,
+  "mmix": null,
+  "mmp": null,
+  "modconf": null,
+  "model": null,
+  "modsim3": null,
+  "modula2": null,
+  "modula3": null,
+  "monk": null,
+  "moo": null,
+  "mp": null,
+  "mplayerconf": null,
+  "mrxvtrc": null,
+  "msidl": null,
+  "msmessages": null,
+  "msql": null,
+  "mupad": null,
+  "mush": null,
+  "muttrc": null,
+  "mysql": null,
+  "named": null,
+  "nanorc": null,
+  "nasm": null,
+  "nastran": null,
+  "natural": null,
+  "ncf": null,
+  "netrc": null,
+  "netrw": null,
+  "nosyntax": null,
+  "nqc": null,
+  "nroff": null,
+  "nsis": null,
+  "obj": null,
+  "objc": "Objective-C",
+  "objcpp": "Objective-C++",
+  "ocaml": "OCaml",
+  "occam": null,
+  "omnimark": null,
+  "openroad": null,
+  "opl": null,
+  "ora": null,
+  "pamconf": null,
+  "papp": null,
+  "pascal": null,
+  "passwd": null,
+  "pcap": null,
+  "pccts": null,
+  "pdf": null,
+  "perl": "Perl",
+  "perl6": "Perl",
+  "pf": null,
+  "pfmain": null,
+  "php": "PHP",
+  "phtml": "PHP",
+  "pic": null,
+  "pike": null,
+  "pilrc": null,
+  "pine": null,
+  "pinfo": null,
+  "plaintex": null,
+  "plm": null,
+  "plp": null,
+  "plsql": null,
+  "po": null,
+  "pod": null,
+  "postscr": null,
+  "pov": null,
+  "povini": null,
+  "ppd": null,
+  "ppwiz": null,
+  "prescribe": null,
+  "privoxy": null,
+  "procmail": null,
+  "progress": null,
+  "prolog": "Prolog",
+  "promela": null,
+  "protocols": null,
+  "psf": null,
+  "ptcap": null,
+  "purifylog": null,
+  "pyrex": null,
+  "python": "Python",
+  "qf": null,
+  "quake": null,
+  "r": "R",
+  "racc": null,
+  "radiance": null,
+  "ratpoison": null,
+  "rc": null,
+  "rcs": null,
+  "rcslog": null,
+  "readline": null,
+  "rebol": null,
+  "registry": null,
+  "remind": null,
+  "resolv": null,
+  "reva": null,
+  "rexx": null,
+  "rhelp": null,
+  "rib": null,
+  "rnc": null,
+  "rnoweb": null,
+  "robots": null,
+  "rpcgen": null,
+  "rpl": null,
+  "rst": null,
+  "rtf": null,
+  "ruby": "Ruby",
+  "samba": null,
+  "sas": null,
+  "sass": "Sass",
+  "sather": null,
+  "scheme": "Scheme",
+  "scilab": null,
+  "screen": null,
+  "scss": "SCSS",
+  "sd": null,
+  "sdc": null,
+  "sdl": null,
+  "sed": null,
+  "sendpr": null,
+  "sensors": null,
+  "services": null,
+  "setserial": null,
+  "sgml": null,
+  "sgmldecl": null,
+  "sgmllnx": null,
+  "sh": null,
+  "sicad": null,
+  "sieve": null,
+  "simula": null,
+  "sinda": null,
+  "sindacmp": null,
+  "sindaout": null,
+  "sisu": null,
+  "skill": "SKILL",
+  "sl": null,
+  "slang": null,
+  "slice": null,
+  "slpconf": null,
+  "slpreg": null,
+  "slpspi": null,
+  "slrnrc": null,
+  "slrnsc": null,
+  "sm": null,
+  "smarty": null,
+  "smcl": null,
+  "smil": null,
+  "smith": null,
+  "sml": null,
+  "snnsnet": null,
+  "snnspat": null,
+  "snnsres": null,
+  "snobol4": null,
+  "spec": null,
+  "specman": null,
+  "spice": null,
+  "splint": null,
+  "spup": null,
+  "spyce": null,
+  "sql": null,
+  "sqlanywhere": null,
+  "sqlforms": null,
+  "sqlinformix": null,
+  "sqlj": null,
+  "sqloracle": null,
+  "sqr": null,
+  "squid": null,
+  "sshconfig": null,
+  "sshdconfig": null,
+  "st": null,
+  "stata": null,
+  "stp": null,
+  "strace": null,
+  "sudoers": null,
+  "svg": null,
+  "svn": null,
+  "syncolor": null,
+  "synload": null,
+  "syntax": null,
+  "sysctl": null,
+  "tads": null,
+  "tags": null,
+  "tak": null,
+  "takcmp": null,
+  "takout": null,
+  "tar": null,
+  "taskdata": null,
+  "taskedit": null,
+  "tasm": null,
+  "tcl": null,
+  "tcsh": null,
+  "terminfo": null,
+  "tex": null,
+  "texinfo": null,
+  "texmf": null,
+  "tf": null,
+  "tidy": null,
+  "tilde": null,
+  "tli": null,
+  "tpp": null,
+  "trasys": null,
+  "trustees": null,
+  "tsalt": null,
+  "tsscl": null,
+  "tssgm": null,
+  "tssop": null,
+  "uc": null,
+  "udevconf": null,
+  "udevperm": null,
+  "udevrules": null,
+  "uil": null,
+  "updatedb": null,
+  "valgrind": null,
+  "vb": "VB.net",
+  "vera": null,
+  "verilog": null,
+  "verilogams": null,
+  "vgrindefs": null,
+  "vhdl": null,
+  "vim": "VimL",
+  "viminfo": null,
+  "virata": null,
+  "vmasm": null,
+  "voscm": null,
+  "vrml": null,
+  "vsejcl": null,
+  "wdiff": null,
+  "web": null,
+  "webmacro": null,
+  "wget": null,
+  "winbatch": null,
+  "wml": null,
+  "wsh": null,
+  "wsml": null,
+  "wvdial": null,
+  "xbl": null,
+  "xdefaults": null,
+  "xf86conf": null,
+  "xhtml": "HTML",
+  "xinetd": null,
+  "xkb": null,
+  "xmath": null,
+  "xml": "XML",
+  "xmodmap": null,
+  "xpm": null,
+  "xpm2": null,
+  "xquery": null,
+  "xs": null,
+  "xsd": null,
+  "xslt": null,
+  "xxd": null,
+  "yacc": null,
+  "yaml": "YAML",
+  "z8a": null,
+  "zsh": null
+}

packages/wakatime/logger.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.logger
+    ~~~~~~~~~~~~~~~
+
+    Provides the configured logger for writing JSON to the log file.
+
+    :copyright: (c) 2013 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import logging
+import os
+import traceback
+
+from .compat import u
+from .packages.requests.packages import urllib3
+try:
+    from collections import OrderedDict  # pragma: nocover
+except ImportError:  # pragma: nocover
+    from .packages.ordereddict import OrderedDict
+try:
+    from .packages import simplejson as json  # pragma: nocover
+except (ImportError, SyntaxError):  # pragma: nocover
+    import json
+
+
+class JsonFormatter(logging.Formatter):
+
+    def setup(self, timestamp, is_write, entity, version, plugin, verbose,
+              warnings=False):
+        self.timestamp = timestamp
+        self.is_write = is_write
+        self.entity = entity
+        self.version = version
+        self.plugin = plugin
+        self.verbose = verbose
+        self.warnings = warnings
+
+    def format(self, record, *args):
+        data = OrderedDict([
+            ('now', self.formatTime(record, self.datefmt)),
+        ])
+        data['version'] = u(self.version)
+        if self.plugin:
+            data['plugin'] = u(self.plugin)
+        data['time'] = self.timestamp
+        if self.verbose:
+            data['caller'] = u(record.pathname)
+            data['lineno'] = record.lineno
+            if self.is_write:
+                data['is_write'] = self.is_write
+            data['file'] = u(self.entity)
+        data['level'] = record.levelname
+        data['message'] = u(record.getMessage() if self.warnings else record.msg)
+        return json.dumps(data)
+
+    def traceback(self, lvl=None):
+        logger = logging.getLogger('WakaTime')
+        if not lvl:
+            lvl = logger.getEffectiveLevel()
+        logger.log(lvl, traceback.format_exc())
+
+
+def set_log_level(logger, args):
+    level = logging.WARN
+    if args.verbose:
+        level = logging.DEBUG
+    logger.setLevel(level)
+
+
+def setup_logging(args, version):
+    urllib3.disable_warnings()
+    logger = logging.getLogger('WakaTime')
+    for handler in logger.handlers:
+        logger.removeHandler(handler)
+    set_log_level(logger, args)
+    logfile = args.logfile
+    if not logfile:
+        logfile = '~/.wakatime.log'
+    handler = logging.FileHandler(os.path.expanduser(logfile))
+    formatter = JsonFormatter(datefmt='%Y/%m/%d %H:%M:%S %z')
+    formatter.setup(
+        timestamp=args.timestamp,
+        is_write=args.is_write,
+        entity=args.entity,
+        version=version,
+        plugin=args.plugin,
+        verbose=args.verbose,
+    )
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+    # add custom traceback logging method
+    logger.traceback = formatter.traceback
+
+    warnings_formatter = JsonFormatter(datefmt='%Y/%m/%d %H:%M:%S %z')
+    warnings_formatter.setup(
+        timestamp=args.timestamp,
+        is_write=args.is_write,
+        entity=args.entity,
+        version=version,
+        plugin=args.plugin,
+        verbose=args.verbose,
+        warnings=True,
+    )
+    warnings_handler = logging.FileHandler(os.path.expanduser(logfile))
+    warnings_handler.setFormatter(warnings_formatter)
+    logging.getLogger('py.warnings').addHandler(warnings_handler)
+    try:
+        logging.captureWarnings(True)
+    except AttributeError:  # pragma: nocover
+        pass  # Python >= 2.7 is needed to capture warnings
+
+    return logger

packages/wakatime/main.py

@@ -0,0 +1,323 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.main
+    ~~~~~~~~~~~~~
+
+    Module entry point.
+
+    :copyright: (c) 2013 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from __future__ import print_function
+
+import base64
+import logging
+import os
+import re
+import sys
+import traceback
+import socket
+
+pwd = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.dirname(pwd))
+sys.path.insert(0, os.path.join(pwd, 'packages'))
+
+from .__about__ import __version__
+from .arguments import parseArguments
+from .compat import u, is_py3
+from .constants import (
+    API_ERROR,
+    AUTH_ERROR,
+    CONFIG_FILE_PARSE_ERROR,
+    SUCCESS,
+    UNKNOWN_ERROR,
+    MALFORMED_HEARTBEAT_ERROR,
+)
+from .logger import setup_logging
+from .offlinequeue import Queue
+from .packages import requests
+from .packages.requests.exceptions import RequestException
+from .project import get_project_info
+from .session_cache import SessionCache
+from .stats import get_file_stats
+from .utils import get_user_agent, should_exclude, format_file_path
+try:
+    from .packages import simplejson as json  # pragma: nocover
+except (ImportError, SyntaxError):  # pragma: nocover
+    import json
+from .packages import tzlocal
+
+
+log = logging.getLogger('WakaTime')
+
+
+def send_heartbeat(project=None, branch=None, hostname=None, stats={}, key=None,
+                   entity=None, timestamp=None, is_write=None, plugin=None,
+                   offline=None, entity_type='file', hidefilenames=None,
+                   proxy=None, api_url=None, timeout=None, **kwargs):
+    """Sends heartbeat as POST request to WakaTime api server.
+
+    Returns `SUCCESS` when heartbeat was sent, otherwise returns an
+    error code constant.
+    """
+
+    if not api_url:
+        api_url = 'https://api.wakatime.com/api/v1/heartbeats'
+    if not timeout:
+        timeout = 60
+    log.debug('Sending heartbeat to api at %s' % api_url)
+    data = {
+        'time': timestamp,
+        'entity': entity,
+        'type': entity_type,
+    }
+    if hidefilenames and entity is not None and entity_type == 'file':
+        for pattern in hidefilenames:
+            try:
+                compiled = re.compile(pattern, re.IGNORECASE)
+                if compiled.search(entity):
+                    extension = u(os.path.splitext(data['entity'])[1])
+                    data['entity'] = u('HIDDEN{0}').format(extension)
+                    break
+            except re.error as ex:
+                log.warning(u('Regex error ({msg}) for include pattern: {pattern}').format(
+                    msg=u(ex),
+                    pattern=u(pattern),
+                ))
+    if stats.get('lines'):
+        data['lines'] = stats['lines']
+    if stats.get('language'):
+        data['language'] = stats['language']
+    if stats.get('dependencies'):
+        data['dependencies'] = stats['dependencies']
+    if stats.get('lineno'):
+        data['lineno'] = stats['lineno']
+    if stats.get('cursorpos'):
+        data['cursorpos'] = stats['cursorpos']
+    if is_write:
+        data['is_write'] = is_write
+    if project:
+        data['project'] = project
+    if branch:
+        data['branch'] = branch
+    log.debug(data)
+
+    # setup api request
+    request_body = json.dumps(data)
+    api_key = u(base64.b64encode(str.encode(key) if is_py3 else key))
+    auth = u('Basic {api_key}').format(api_key=api_key)
+    headers = {
+        'User-Agent': get_user_agent(plugin),
+        'Content-Type': 'application/json',
+        'Accept': 'application/json',
+        'Authorization': auth,
+    }
+    if hostname:
+        headers['X-Machine-Name'] = u(hostname).encode('utf-8')
+
+    # add Olson timezone to request
+    try:
+        tz = tzlocal.get_localzone()
+    except:
+        tz = None
+    if tz:
+        headers['TimeZone'] = u(tz.zone).encode('utf-8')
+
+    session_cache = SessionCache()
+    session = session_cache.get()
+
+    proxies = {}
+    if proxy:
+        if '\\' in proxy:
+            from .packages.requests_ntlm import HttpNtlmAuth
+            username = proxy.rsplit(':', 1)
+            password = ''
+            if len(username) == 2:
+                password = username[1]
+            username = username[0]
+            session.auth = HttpNtlmAuth(username, password, session)
+        else:
+            proxies['https'] = proxy
+
+    # log time to api
+    response = None
+    try:
+        response = session.post(api_url, data=request_body, headers=headers,
+                                proxies=proxies, timeout=timeout)
+    except RequestException:
+        exception_data = {
+            sys.exc_info()[0].__name__: u(sys.exc_info()[1]),
+        }
+        if log.isEnabledFor(logging.DEBUG):
+            exception_data['traceback'] = traceback.format_exc()
+        if offline:
+            queue = Queue()
+            queue.push(data, json.dumps(stats), plugin)
+            if log.isEnabledFor(logging.DEBUG):
+                log.warn(exception_data)
+        else:
+            log.error(exception_data)
+
+    except:  # delete cached session when requests raises unknown exception
+        exception_data = {
+            sys.exc_info()[0].__name__: u(sys.exc_info()[1]),
+            'traceback': traceback.format_exc(),
+        }
+        if offline:
+            queue = Queue()
+            queue.push(data, json.dumps(stats), plugin)
+            log.warn(exception_data)
+        session_cache.delete()
+
+    else:
+        code = response.status_code if response is not None else None
+        content = response.text if response is not None else None
+        if code == requests.codes.created or code == requests.codes.accepted:
+            log.debug({
+                'response_code': code,
+            })
+            session_cache.save(session)
+            return SUCCESS
+        if offline:
+            if code != 400:
+                queue = Queue()
+                queue.push(data, json.dumps(stats), plugin)
+                if code == 401:
+                    log.error({
+                        'response_code': code,
+                        'response_content': content,
+                    })
+                    session_cache.delete()
+                    return AUTH_ERROR
+                elif log.isEnabledFor(logging.DEBUG):
+                    log.warn({
+                        'response_code': code,
+                        'response_content': content,
+                    })
+            else:
+                log.error({
+                    'response_code': code,
+                    'response_content': content,
+                })
+        else:
+            log.error({
+                'response_code': code,
+                'response_content': content,
+            })
+    session_cache.delete()
+    return API_ERROR
+
+
+def sync_offline_heartbeats(args, hostname):
+    """Sends all heartbeats which were cached in the offline Queue."""
+
+    queue = Queue()
+    while True:
+        heartbeat = queue.pop()
+        if heartbeat is None:
+            break
+        status = send_heartbeat(
+            project=heartbeat['project'],
+            entity=heartbeat['entity'],
+            timestamp=heartbeat['time'],
+            branch=heartbeat['branch'],
+            hostname=hostname,
+            stats=json.loads(heartbeat['stats']),
+            key=args.key,
+            is_write=heartbeat['is_write'],
+            plugin=heartbeat['plugin'],
+            offline=args.offline,
+            hidefilenames=args.hidefilenames,
+            entity_type=heartbeat['type'],
+            proxy=args.proxy,
+            api_url=args.api_url,
+            timeout=args.timeout,
+        )
+        if status != SUCCESS:
+            if status == AUTH_ERROR:
+                return AUTH_ERROR
+            break
+    return SUCCESS
+
+
+def process_heartbeat(args, configs, hostname, heartbeat):
+    exclude = should_exclude(heartbeat['entity'], args.include, args.exclude)
+    if exclude is not False:
+        log.debug(u('Skipping because matches exclude pattern: {pattern}').format(
+            pattern=u(exclude),
+        ))
+        return SUCCESS
+
+    if heartbeat.get('entity_type') not in ['file', 'domain', 'app']:
+        heartbeat['entity_type'] = 'file'
+
+    if heartbeat['entity_type'] == 'file':
+        heartbeat['entity'] = format_file_path(heartbeat['entity'])
+
+    if heartbeat['entity_type'] != 'file' or os.path.isfile(heartbeat['entity']):
+
+        stats = get_file_stats(heartbeat['entity'],
+                                entity_type=heartbeat['entity_type'],
+                                lineno=heartbeat.get('lineno'),
+                                cursorpos=heartbeat.get('cursorpos'),
+                                plugin=args.plugin,
+                                language=heartbeat.get('language'))
+
+        project = heartbeat.get('project') or heartbeat.get('alternate_project')
+        branch = None
+        if heartbeat['entity_type'] == 'file':
+            project, branch = get_project_info(configs, heartbeat)
+
+        heartbeat['project'] = project
+        heartbeat['branch'] = branch
+        heartbeat['stats'] = stats
+        heartbeat['hostname'] = hostname
+        heartbeat['timeout'] = args.timeout
+        heartbeat['key'] = args.key
+        heartbeat['plugin'] = args.plugin
+        heartbeat['offline'] = args.offline
+        heartbeat['hidefilenames'] = args.hidefilenames
+        heartbeat['proxy'] = args.proxy
+        heartbeat['api_url'] = args.api_url
+
+        return send_heartbeat(**heartbeat)
+
+    else:
+        log.debug('File does not exist; ignoring this heartbeat.')
+        return SUCCESS
+
+
+def execute(argv=None):
+    if argv:
+        sys.argv = ['wakatime'] + argv
+
+    args, configs = parseArguments()
+    if configs is None:
+        return CONFIG_FILE_PARSE_ERROR
+
+    setup_logging(args, __version__)
+
+    try:
+
+        hostname = args.hostname or socket.gethostname()
+
+        heartbeat = vars(args)
+        retval = process_heartbeat(args, configs, hostname, heartbeat)
+
+        if args.extra_heartbeats:
+            try:
+                for heartbeat in json.loads(sys.stdin.readline()):
+                    retval = process_heartbeat(args, configs, hostname, heartbeat)
+            except json.JSONDecodeError:
+                retval = MALFORMED_HEARTBEAT_ERROR
+
+        if retval == SUCCESS:
+            retval = sync_offline_heartbeats(args, hostname)
+
+        return retval
+
+    except:
+        log.traceback(logging.ERROR)
+        print(traceback.format_exc())
+        return UNKNOWN_ERROR

packages/wakatime/offlinequeue.py

@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.offlinequeue
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Queue for saving heartbeats while offline.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+
+import logging
+import os
+from time import sleep
+
+try:
+    import sqlite3
+    HAS_SQL = True
+except ImportError:  # pragma: nocover
+    HAS_SQL = False
+
+from .compat import u
+
+
+log = logging.getLogger('WakaTime')
+
+
+class Queue(object):
+    db_file = os.path.join(os.path.expanduser('~'), '.wakatime.db')
+    table_name = 'heartbeat_1'
+
+    def get_db_file(self):
+        return self.db_file
+
+    def connect(self):
+        conn = sqlite3.connect(self.get_db_file(), isolation_level=None)
+        c = conn.cursor()
+        c.execute('''CREATE TABLE IF NOT EXISTS {0} (
+            entity text,
+            type text,
+            time real,
+            project text,
+            branch text,
+            is_write integer,
+            stats text,
+            misc text,
+            plugin text)
+        '''.format(self.table_name))
+        return (conn, c)
+
+    def push(self, data, stats, plugin, misc=None):
+        if not HAS_SQL:  # pragma: nocover
+            return
+        try:
+            conn, c = self.connect()
+            heartbeat = {
+                'entity': u(data.get('entity')),
+                'type': u(data.get('type')),
+                'time': data.get('time'),
+                'project': u(data.get('project')),
+                'branch': u(data.get('branch')),
+                'is_write': 1 if data.get('is_write') else 0,
+                'stats': u(stats),
+                'misc': u(misc),
+                'plugin': u(plugin),
+            }
+            c.execute('INSERT INTO {0} VALUES (:entity,:type,:time,:project,:branch,:is_write,:stats,:misc,:plugin)'.format(self.table_name), heartbeat)
+            conn.commit()
+            conn.close()
+        except sqlite3.Error:
+            log.traceback()
+
+    def pop(self):
+        if not HAS_SQL:  # pragma: nocover
+            return None
+        tries = 3
+        wait = 0.1
+        heartbeat = None
+        try:
+            conn, c = self.connect()
+        except sqlite3.Error:
+            log.traceback(logging.DEBUG)
+            return None
+        loop = True
+        while loop and tries > -1:
+            try:
+                c.execute('BEGIN IMMEDIATE')
+                c.execute('SELECT * FROM {0} LIMIT 1'.format(self.table_name))
+                row = c.fetchone()
+                if row is not None:
+                    values = []
+                    clauses = []
+                    index = 0
+                    for row_name in ['entity', 'type', 'time', 'project', 'branch', 'is_write']:
+                        if row[index] is not None:
+                            clauses.append('{0}=?'.format(row_name))
+                            values.append(row[index])
+                        else:  # pragma: nocover
+                            clauses.append('{0} IS NULL'.format(row_name))
+                        index += 1
+                    if len(values) > 0:
+                        c.execute('DELETE FROM {0} WHERE {1}'.format(self.table_name, ' AND '.join(clauses)), values)
+                    else:  # pragma: nocover
+                        c.execute('DELETE FROM {0} WHERE {1}'.format(self.table_name, ' AND '.join(clauses)))
+                conn.commit()
+                if row is not None:
+                    heartbeat = {
+                        'entity': row[0],
+                        'type': row[1],
+                        'time': row[2],
+                        'project': row[3],
+                        'branch': row[4],
+                        'is_write': True if row[5] is 1 else False,
+                        'stats': row[6],
+                        'misc': row[7],
+                        'plugin': row[8],
+                    }
+                loop = False
+            except sqlite3.Error:  # pragma: nocover
+                log.traceback(logging.DEBUG)
+                sleep(wait)
+                tries -= 1
+        try:
+            conn.close()
+        except sqlite3.Error:  # pragma: nocover
+            log.traceback(logging.DEBUG)
+        return heartbeat

packages/wakatime/packages/argparse.py

@@ -61,7 +61,12 @@ considered public as object names -- the API of the formatter objects is
 still considered an implementation detail.)
 """
 
-__version__ = '1.2.1'
+__version__ = '1.3.0'  # we use our own version number independant of the
+                       # one in stdlib and we release this on pypi.
+
+__external_lib__ = True  # to make sure the tests really test THIS lib,
+                         # not the builtin one in Python stdlib
+
 __all__ = [
     'ArgumentParser',
     'ArgumentError',
@@ -1045,9 +1050,13 @@ class _SubParsersAction(Action):
 
     class _ChoicesPseudoAction(Action):
 
-        def __init__(self, name, help):
+        def __init__(self, name, aliases, help):
+            metavar = dest = name
+            if aliases:
+                metavar += ' (%s)' % ', '.join(aliases)
             sup = super(_SubParsersAction._ChoicesPseudoAction, self)
-            sup.__init__(option_strings=[], dest=name, help=help)
+            sup.__init__(option_strings=[], dest=dest, help=help,
+                        metavar=metavar)
 
     def __init__(self,
                  option_strings,
@@ -1075,15 +1084,22 @@ class _SubParsersAction(Action):
         if kwargs.get('prog') is None:
             kwargs['prog'] = '%s %s' % (self._prog_prefix, name)
 
+        aliases = kwargs.pop('aliases', ())
+
         # create a pseudo-action to hold the choice help
         if 'help' in kwargs:
             help = kwargs.pop('help')
-            choice_action = self._ChoicesPseudoAction(name, help)
+            choice_action = self._ChoicesPseudoAction(name, aliases, help)
             self._choices_actions.append(choice_action)
 
         # create the parser and add it to the map
         parser = self._parser_class(**kwargs)
         self._name_parser_map[name] = parser
+
+        # make parser available under aliases also
+        for alias in aliases:
+            self._name_parser_map[alias] = parser
+
         return parser
 
     def _get_subactions(self):

packages/wakatime/packages/ntlm_auth/U32.py

@@ -0,0 +1,156 @@
+# This file is part of 'NTLM Authorization Proxy Server' http://sourceforge.net/projects/ntlmaps/
+# Copyright 2001 Dmitry A. Rozmanov <dima@xenon.spb.ru>
+#
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+from __future__ import division
+import six
+
+C = 0x1000000000
+
+
+def norm(n):
+    return n & 0xFFFFFFFF
+
+
+class U32:
+    v = 0
+
+    def __init__(self, value=0):
+        if not isinstance(value, six.integer_types):
+            value = six.byte2int(value)
+
+        self.v = C + norm(abs(int(value)))
+
+    def set(self, value=0):
+        self.v = C + norm(abs(int(value)))
+
+    def __repr__(self):
+        return hex(norm(self.v))
+
+    def __long__(self):
+        return int(norm(self.v))
+
+    def __int__(self):
+        return int(norm(self.v))
+
+    def __chr__(self):
+        return chr(norm(self.v))
+
+    def __add__(self, b):
+        r = U32()
+        r.v = C + norm(self.v + b.v)
+        return r
+
+    def __sub__(self, b):
+        r = U32()
+        if self.v < b.v:
+            r.v = C + norm(0x100000000 - (b.v - self.v))
+        else:
+            r.v = C + norm(self.v - b.v)
+        return r
+
+    def __mul__(self, b):
+        r = U32()
+        r.v = C + norm(self.v * b.v)
+        return r
+
+    def __div__(self, b):
+        r = U32()
+        r.v = C + (norm(self.v) // norm(b.v))
+        return r
+
+    def __truediv__(self, b):
+        r = U32()
+        r.v = C + (norm(self.v) / norm(b.v))
+        return r
+
+    def __mod__(self, b):
+        r = U32()
+        r.v = C + (norm(self.v) % norm(b.v))
+        return r
+
+    def __neg__(self):
+        return U32(self.v)
+
+    def __pos__(self):
+        return U32(self.v)
+
+    def __abs__(self):
+        return U32(self.v)
+
+    def __invert__(self):
+        r = U32()
+        r.v = C + norm(~self.v)
+        return r
+
+    def __lshift__(self, b):
+        r = U32()
+        r.v = C + norm(self.v << b)
+        return r
+
+    def __rshift__(self, b):
+        r = U32()
+        r.v = C + (norm(self.v) >> b)
+        return r
+
+    def __and__(self, b):
+        r = U32()
+        r.v = C + norm(self.v & b.v)
+        return r
+
+    def __or__(self, b):
+        r = U32()
+        r.v = C + norm(self.v | b.v)
+        return r
+
+    def __xor__(self, b):
+        r = U32()
+        r.v = C + norm(self.v ^ b.v)
+        return r
+
+    def __not__(self):
+        return U32(not norm(self.v))
+
+    def truth(self):
+        return norm(self.v)
+
+    def __cmp__(self, b):
+        if norm(self.v) > norm(b.v):
+            return 1
+        elif norm(self.v) < norm(b.v):
+            return -1
+        else:
+            return 0
+
+    def __lt__(self, other):
+        return self.v < other.v
+
+    def __gt__(self, other):
+        return self.v > other.v
+
+    def __eq__(self, other):
+        return self.v == other.v
+
+    def __le__(self, other):
+        return self.v <= other.v
+
+    def __ge__(self, other):
+        return self.v >= other.v
+
+    def __ne__(self, other):
+        return self.v != other.v
+
+    def __nonzero__(self):
+        return norm(self.v)

packages/wakatime/packages/ntlm_auth/__init__.py

@@ -0,0 +1,3 @@
+from . import ntlm, session_security
+
+__all__ = ('ntlm', 'session_security')

packages/wakatime/packages/ntlm_auth/compute_hash.py

@@ -0,0 +1,80 @@
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+import binascii
+import hashlib
+import hmac
+import re
+from ntlm_auth import des
+
+
+def _lmowfv1(password):
+    """
+    [MS-NLMP] v28.0 2016-07-14
+
+    3.3.1 NTLM v1 Authentication
+    Same function as LMOWFv1 in document to create a one way hash of the password. Only
+    used in NTLMv1 auth without session security
+
+    :param password: The password of the user we are trying to authenticate with
+    :return res: A Lan Manager hash of the password supplied
+    """
+
+    # fix the password length to 14 bytes
+    password = password.upper()
+    lm_pw = password[0:14]
+
+    # do hash
+    magic_str = b"KGS!@#$%"  # page 56 in [MS-NLMP v28.0]
+
+    res = b''
+    dobj = des.DES(lm_pw[0:7])
+    res = res + dobj.encrypt(magic_str)
+
+    dobj = des.DES(lm_pw[7:14])
+    res = res + dobj.encrypt(magic_str)
+
+    return res
+
+def _ntowfv1(password):
+    """
+    [MS-NLMP] v28.0 2016-07-14
+
+    3.3.1 NTLM v1 Authentication
+    Same function as NTOWFv1 in document to create a one way hash of the password. Only
+    used in NTLMv1 auth without session security
+
+    :param password: The password of the user we are trying to authenticate with
+    :return digest: An NT hash of the password supplied
+    """
+
+    digest = hashlib.new('md4', password.encode('utf-16le')).digest()
+    return digest
+
+def _ntowfv2(user_name, password, domain_name):
+    """
+    [MS-NLMP] v28.0 2016-07-14
+
+    3.3.2 NTLM v2 Authentication
+    Same function as NTOWFv2 (and LMOWFv2) in document to create a one way hash of the password.
+    This combines some extra security features over the v1 calculations used in NTLMv2 auth.
+
+    :param user_name: The user name of the user we are trying to authenticate with
+    :param password: The password of the user we are trying to authenticate with
+    :param domain_name: The domain name of the user account we are authenticated with
+    :return digest: An NT hash of the parameters supplied
+    """
+    digest = _ntowfv1(password)
+    digest = hmac.new(digest, (user_name.upper() + domain_name).encode('utf-16le')).digest()
+
+    return digest

packages/wakatime/packages/ntlm_auth/compute_keys.py

@@ -0,0 +1,138 @@
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+import binascii
+import hashlib
+import hmac
+from ntlm_auth import des
+from ntlm_auth.constants import NegotiateFlags
+
+def _get_exchange_key_ntlm_v1(negotiate_flags, session_base_key, server_challenge, lm_challenge_response, lm_hash):
+    """
+    [MS-NLMP] v28.0 2016-07-14
+
+    4.3.5.1 KXKEY
+    Calculates the Key Exchange Key for NTLMv1 authentication. Used for signing and sealing messages
+
+    @param negotiate_flags:
+    @param session_base_key: A session key calculated from the user password challenge
+    @param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
+    @param lm_challenge_response: The LmChallengeResponse value computed in ComputeResponse
+    @param lm_hash: The LMOWF computed in Compute Response
+    @return key_exchange_key: The Key Exchange Key (KXKEY) used to sign and seal messages and compute the ExportedSessionKey
+    """
+    if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY:
+        key_exchange_key = hmac.new(session_base_key, server_challenge + lm_challenge_response[:8]).digest()
+    elif negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_LM_KEY:
+        des_handler = des.DES(lm_hash[:7])
+        first_des = des_handler.encrypt(lm_challenge_response[:8])
+        des_handler = des.DES(lm_hash[7:8] + binascii.unhexlify('bdbdbdbdbdbdbd'))
+        second_des = des_handler.encrypt(lm_challenge_response[:8])
+
+        key_exchange_key = first_des + second_des
+    elif negotiate_flags & NegotiateFlags.NTLMSSP_REQUEST_NON_NT_SESSION_KEY:
+        key_exchange_key = lm_hash[:8] + b'\0' * 8
+    else:
+        key_exchange_key = session_base_key
+
+    return key_exchange_key
+
+def _get_exchange_key_ntlm_v2(session_base_key):
+    """
+    [MS-NLMP] v28.0 2016-07-14
+
+    4.3.5.1 KXKEY
+    Calculates the Key Exchange Key for NTLMv2 authentication. Used for signing and sealing messages.
+    According to docs, 'If NTLM v2 is used, KeyExchangeKey MUST be set to the given 128-bit SessionBaseKey
+
+    @param session_base_key: A session key calculated from the user password challenge
+    @return key_exchange_key: The Key Exchange Key (KXKEY) used to sign and seal messages
+    """
+    return session_base_key
+
+def get_sign_key(exported_session_key, magic_constant):
+    """
+    3.4.5.2 SIGNKEY
+
+    @param exported_session_key: A 128-bit session key used to derive signing and sealing keys
+    @param magic_constant: A constant value set in the MS-NLMP documentation (constants.SignSealConstants)
+    @return sign_key: Key used to sign messages
+    """
+
+    sign_key = hashlib.md5(exported_session_key + magic_constant).digest()
+
+    return sign_key
+
+def get_seal_key(negotiate_flags, exported_session_key, magic_constant):
+    """
+    3.4.5.3. SEALKEY
+    Main method to use to calculate the seal_key used to seal (encrypt) messages. This will determine
+    the correct method below to use based on the compatibility flags set and should be called instead
+    of the others
+
+    @param exported_session_key: A 128-bit session key used to derive signing and sealing keys
+    @param negotiate_flags: The negotiate_flags structure sent by the server
+    @param magic_constant: A constant value set in the MS-NLMP documentation (constants.SignSealConstants)
+    @return seal_key: Key used to seal messages
+    """
+
+    if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY:
+        seal_key = _get_seal_key_ntlm2(negotiate_flags, exported_session_key, magic_constant)
+    elif negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_LM_KEY:
+        seal_key = _get_seal_key_ntlm1(negotiate_flags, exported_session_key)
+    else:
+        seal_key = exported_session_key
+
+    return seal_key
+
+def _get_seal_key_ntlm1(negotiate_flags, exported_session_key):
+    """
+    3.4.5.3 SEALKEY
+    Calculates the seal_key used to seal (encrypt) messages. This for authentication where
+    NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY has not been negotiated. Will weaken the keys
+    if NTLMSSP_NEGOTIATE_56 is not negotiated it will default to the 40-bit key
+
+    @param negotiate_flags: The negotiate_flags structure sent by the server
+    @param exported_session_key: A 128-bit session key used to derive signing and sealing keys
+    @return seal_key: Key used to seal messages
+    """
+    if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_56:
+        seal_key = exported_session_key[:7] + binascii.unhexlify('a0')
+    else:
+        seal_key = exported_session_key[:5] + binascii.unhexlify('e538b0')
+
+    return seal_key
+
+def _get_seal_key_ntlm2(negotiate_flags, exported_session_key, magic_constant):
+    """
+    3.4.5.3 SEALKEY
+    Calculates the seal_key used to seal (encrypt) messages. This for authentication where
+    NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY has been negotiated. Will weaken the keys
+    if NTLMSSP_NEGOTIATE_128 is not negotiated, will try NEGOTIATE_56 and then will default
+    to the 40-bit key
+
+    @param negotiate_flags: The negotiate_flags structure sent by the server
+    @param exported_session_key: A 128-bit session key used to derive signing and sealing keys
+    @param magic_constant: A constant value set in the MS-NLMP documentation (constants.SignSealConstants)
+    @return seal_key: Key used to seal messages
+    """
+    if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_128:
+        seal_key = exported_session_key
+    elif negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_56:
+        seal_key = exported_session_key[:7]
+    else:
+        seal_key = exported_session_key[:5]
+
+    seal_key = hashlib.md5(seal_key + magic_constant).digest()
+
+    return seal_key

packages/wakatime/packages/ntlm_auth/compute_response.py

@@ -0,0 +1,397 @@
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+import base64
+import calendar
+import hashlib
+import hmac
+import os
+import struct
+import time
+import ntlm_auth.compute_hash as comphash
+import ntlm_auth.compute_keys as compkeys
+from ntlm_auth import des
+from ntlm_auth.constants import NegotiateFlags, AvFlags
+from ntlm_auth.gss_channel_bindings import GssChannelBindingsStruct
+from ntlm_auth.target_info import TargetInfo
+
+class ComputeResponse():
+    """
+        Constructor for the response computations. This class will compute the various
+        nt and lm challenge responses.
+
+        :param user_name: The user name of the user we are trying to authenticate with
+        :param password: The password of the user we are trying to authenticate with
+        :param domain_name: The domain name of the user account we are authenticated with, default is None
+        :param challenge_message: A ChallengeMessage object that was received from the server after the negotiate_message
+        :param ntlm_compatibility: The Lan Manager Compatibility Level, used to determine what NTLM auth version to use, see Ntlm in ntlm.py for more details
+    """
+    def __init__(self, user_name, password, domain_name, challenge_message, ntlm_compatibility):
+        self._user_name = user_name
+        self._password = password
+        self._domain_name = domain_name
+        self._challenge_message = challenge_message
+        self._negotiate_flags = challenge_message.negotiate_flags
+        self._server_challenge = challenge_message.server_challenge
+        self._server_target_info = challenge_message.target_info
+        self._ntlm_compatibility = ntlm_compatibility
+        self._client_challenge = os.urandom(8)
+
+    def get_lm_challenge_response(self):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        3.3.1 - NTLM v1 Authentication
+        3.3.2 - NTLM v2 Authentication
+
+        This method returns the LmChallengeResponse key based on the ntlm_compatibility chosen
+        and the target_info supplied by the CHALLENGE_MESSAGE. It is quite different from what
+        is set in the document as it combines the NTLMv1, NTLM2 and NTLMv2 methods into one
+        and calls separate methods based on the ntlm_compatibility flag chosen.
+
+        :return: response (LmChallengeResponse) - The LM response to the server challenge. Computed by the client
+        """
+        if self._negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY and self._ntlm_compatibility < 3:
+            response = ComputeResponse._get_LMv1_with_session_security_response(self._client_challenge)
+
+        elif 0 <= self._ntlm_compatibility <= 1:
+            response = ComputeResponse._get_LMv1_response(self._password, self._server_challenge)
+        elif self._ntlm_compatibility == 2:
+            # Based on the compatibility level we don't want to use LM responses, ignore the session_base_key as it is returned in nt
+            response, ignore_key = ComputeResponse._get_NTLMv1_response(self._password, self._server_challenge)
+        else:
+            """
+            [MS-NLMP] v28.0 page 45 - 2016-07-14
+
+            3.1.5.12 Client Received a CHALLENGE_MESSAGE from the Server
+            If NTLMv2 authentication is used and the CHALLENGE_MESSAGE TargetInfo field has an MsvAvTimestamp present,
+            the client SHOULD NOT send the LmChallengeResponse and SHOULD send Z(24) instead.
+            """
+
+            response = ComputeResponse._get_LMv2_response(self._user_name, self._password, self._domain_name,
+                                                          self._server_challenge,
+                                                          self._client_challenge)
+            if self._server_target_info is not None:
+                timestamp = self._server_target_info[TargetInfo.MSV_AV_TIMESTAMP]
+                if timestamp is not None:
+                    response = b'\0' * 24
+
+        return response
+
+    def get_nt_challenge_response(self, lm_challenge_response, server_certificate_hash):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        3.3.1 - NTLM v1 Authentication
+        3.3.2 - NTLM v2 Authentication
+
+        This method returns the NtChallengeResponse key based on the ntlm_compatibility chosen
+        and the target_info supplied by the CHALLENGE_MESSAGE. It is quite different from what
+        is set in the document as it combines the NTLMv1, NTLM2 and NTLMv2 methods into one
+        and calls separate methods based on the ntlm_compatibility value chosen.
+
+        :param lm_challenge_response: The LmChallengeResponse calculated beforeand, used to get the key_exchange_key value
+        :param server_certificate_hash: The SHA256 hash of the server certificate (DER encoded) NTLM is authenticated to.
+                                        Used in Channel Binding Tokens if present, default value is None. See
+                                        AuthenticateMessage in messages.py for more details
+        :return response: (NtChallengeResponse) - The NT response to the server challenge. Computed by the client
+        :return session_base_key: (SessionBaseKey) - A session key calculated from the user password challenge
+        :return target_info: (AV_PAIR) - The AV_PAIR structure used in the nt_challenge calculations
+        """
+        if self._negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY and self._ntlm_compatibility < 3:
+            # The compatibility level is less than 3 which means it doesn't support NTLMv2 but we want extended security so use NTLM2 which is different from NTLMv2
+            # [MS-NLMP] - 3.3.1 NTLMv1 Authentication
+            response, session_base_key = ComputeResponse._get_NTLM2_response(self._password, self._server_challenge, self._client_challenge)
+            key_exchange_key = compkeys._get_exchange_key_ntlm_v1(self._negotiate_flags, session_base_key,
+                                                                  self._server_challenge, lm_challenge_response,
+                                                                  comphash._lmowfv1(self._password))
+            target_info = None
+
+        elif 0 <= self._ntlm_compatibility < 3:
+            response, session_base_key = ComputeResponse._get_NTLMv1_response(self._password, self._server_challenge)
+            key_exchange_key = compkeys._get_exchange_key_ntlm_v1(self._negotiate_flags, session_base_key,
+                                                                  self._server_challenge, lm_challenge_response,
+                                                                  comphash._lmowfv1(self._password))
+            target_info = None
+        else:
+            if self._server_target_info is None:
+                target_info = TargetInfo()
+            else:
+                target_info = self._server_target_info
+
+            if target_info[TargetInfo.MSV_AV_TIMESTAMP] is None:
+                timestamp = get_windows_timestamp()
+            else:
+                timestamp = target_info[TargetInfo.MSV_AV_TIMESTAMP][1]
+
+                # [MS-NLMP] If the CHALLENGE_MESSAGE TargetInfo field has an MsvAvTimestamp present, the client SHOULD provide a MIC
+                target_info[TargetInfo.MSV_AV_FLAGS] = struct.pack("<L", AvFlags.MIC_PROVIDED)
+
+            if server_certificate_hash != None:
+                channel_bindings_hash = ComputeResponse._get_channel_bindings_value(server_certificate_hash)
+                target_info[TargetInfo.MSV_AV_CHANNEL_BINDINGS] = channel_bindings_hash
+
+            response, session_base_key = ComputeResponse._get_NTLMv2_response(self._user_name, self._password, self._domain_name,
+                                                 self._server_challenge, self._client_challenge, timestamp, target_info)
+
+            key_exchange_key = compkeys._get_exchange_key_ntlm_v2(session_base_key)
+
+        return response, key_exchange_key, target_info
+
+
+    @staticmethod
+    def _get_LMv1_response(password, server_challenge):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.2.3 LM_RESPONSE
+        The LM_RESPONSE structure defines the NTLM v1 authentication LmChallengeResponse
+        in the AUTHENTICATE_MESSAGE. This response is used only when NTLM v1
+        authentication is configured.
+
+        :param password: The password of the user we are trying to authenticate with
+        :param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
+        :return response: LmChallengeResponse to the server challenge
+        """
+        lm_hash = comphash._lmowfv1(password)
+        response = ComputeResponse._calc_resp(lm_hash, server_challenge)
+
+        return response
+
+    @staticmethod
+    def _get_LMv1_with_session_security_response(client_challenge):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.2.3 LM_RESPONSE
+        The LM_RESPONSE structure defines the NTLM v1 authentication LmChallengeResponse
+        in the AUTHENTICATE_MESSAGE. This response is used only when NTLM v1
+        authentication is configured and NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY is flages.
+
+        :param client_challenge: A random 8-byte response generated by the client for the AUTHENTICATE_MESSAGE
+        :return response: LmChallengeResponse to the server challenge
+        """
+
+        response = client_challenge + b'\0' * 16
+
+        return response
+
+    @staticmethod
+    def _get_LMv2_response(user_name, password, domain_name, server_challenge, client_challenge):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.2.4 LMv2_RESPONSE
+        The LMv2_RESPONSE structure defines the NTLM v2 authentication LmChallengeResponse
+        in the AUTHENTICATE_MESSAGE. This response is used only when NTLM v2
+        authentication is configured.
+
+        :param user_name: The user name of the user we are trying to authenticate with
+        :param password: The password of the user we are trying to authenticate with
+        :param domain_name: The domain name of the user account we are authenticated with
+        :param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
+        :param client_challenge: A random 8-byte response generated by the client for the AUTHENTICATE_MESSAGE
+        :return response: LmChallengeResponse to the server challenge
+        """
+        nt_hash = comphash._ntowfv2(user_name, password, domain_name)
+        lm_hash = hmac.new(nt_hash, (server_challenge + client_challenge)).digest()
+        response = lm_hash + client_challenge
+
+        return response
+
+    @staticmethod
+    def _get_NTLMv1_response(password, server_challenge):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.2.6 NTLM v1 Response: NTLM_RESPONSE
+        The NTLM_RESPONSE strucutre defines the NTLM v1 authentication NtChallengeResponse
+        in the AUTHENTICATE_MESSAGE. This response is only used when NTLM v1 authentication
+        is configured.
+
+        :param password: The password of the user we are trying to authenticate with
+        :param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
+        :return response: NtChallengeResponse to the server_challenge
+        :return session_base_key: A session key calculated from the user password challenge
+        """
+        ntlm_hash = comphash._ntowfv1(password)
+        response = ComputeResponse._calc_resp(ntlm_hash, server_challenge)
+
+        session_base_key = hashlib.new('md4', ntlm_hash).digest()
+
+        return response, session_base_key
+
+    @staticmethod
+    def _get_NTLM2_response(password, server_challenge, client_challenge):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        This name is really misleading as it isn't NTLM v2 authentication rather
+        This authentication is only used when the ntlm_compatibility level is set
+        to a value < 3 (No NTLMv2 auth) but the NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY
+        flag is set in the negotiate flags section. The documentation for computing this
+        value is on page 56 under section 3.3.1 NTLM v1 Authentication
+
+        :param password: The password of the user we are trying to authenticate with
+        :param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
+        :param client_challenge: A random 8-byte response generated by the client for the AUTHENTICATE_MESSAGE
+        :return response: NtChallengeResponse to the server_challenge
+        :return session_base_key: A session key calculated from the user password challenge
+        """
+        ntlm_hash = comphash._ntowfv1(password)
+        nt_session_hash = hashlib.md5(server_challenge + client_challenge).digest()[:8]
+        response = ComputeResponse._calc_resp(ntlm_hash, nt_session_hash[0:8])
+
+        session_base_key = hashlib.new('md4', ntlm_hash).digest()
+
+        return response, session_base_key
+
+    @staticmethod
+    def _get_NTLMv2_response(user_name, password, domain_name, server_challenge, client_challenge, timestamp, target_info):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.2.8 NTLM V2 Response: NTLMv2_RESPONSE
+        The NTLMv2_RESPONSE strucutre defines the NTLMv2 authentication NtChallengeResponse
+        in the AUTHENTICATE_MESSAGE. This response is used only when NTLMv2 authentication
+        is configured.
+
+        The guide on how this is computed is in 3.3.2 NTLM v2 Authentication.
+
+        :param user_name: The user name of the user we are trying to authenticate with
+        :param password: The password of the user we are trying to authenticate with
+        :param domain_name: The domain name of the user account we are authenticated with
+        :param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
+        :param client_challenge: A random 8-byte response generated by the client for the AUTHENTICATE_MESSAGE
+        :param timestamp: An 8-byte timestamp in windows format, 100 nanoseconds since 1601-01-01
+        :param target_info: The target_info structure from the CHALLENGE_MESSAGE with the CBT attached if required
+        :return response: NtChallengeResponse to the server_challenge
+        :return session_base_key: A session key calculated from the user password challenge
+        """
+
+        nt_hash = comphash._ntowfv2(user_name, password, domain_name)
+        temp = ComputeResponse._get_NTLMv2_temp(timestamp, client_challenge, target_info)
+        nt_proof_str = hmac.new(nt_hash, (server_challenge + temp)).digest()
+        response = nt_proof_str + temp
+
+        session_base_key = hmac.new(nt_hash, nt_proof_str).digest()
+
+        return response, session_base_key
+
+    @staticmethod
+    def _get_NTLMv2_temp(timestamp, client_challenge, target_info):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.2.7 NTLMv2_CLIENT_CHALLENGE - variable length
+        The NTLMv2_CLIENT_CHALLENGE structure defines the client challenge in
+        the AUTHENTICATE_MESSAGE. This structure is used only when NTLM v2
+        authentication is configured and is transported in the NTLMv2_RESPONSE
+        structure.
+
+        The method to create this structure is defined in 3.3.2 NTLMv2 Authentication.
+        In this method this variable is known as the temp value. The target_info variable
+        corresponds to the ServerName variable used in that documentation. This is in
+        reality a lot more than just the ServerName and contains the AV_PAIRS structure
+        we need to transport with the message like Channel Binding tokens and others.
+        By default this will be the target_info returned from the CHALLENGE_MESSAGE plus
+        MSV_AV_CHANNEL_BINDINGS if specified otherwise it is a new target_info set with
+        MSV_AV_TIMESTAMP to the current time.
+
+        :param timestamp: An 8-byte timestamp in windows format, 100 nanoseconds since 1601-01-01
+        :param client_challenge: A random 8-byte response generated by the client for the AUTHENTICATE_MESSAGE
+        :param target_info: The target_info structure from the CHALLENGE_MESSAGE with the CBT attached if required
+        :return temp: The CLIENT_CHALLENGE structure that will be added to the NtChallengeResponse structure
+        """
+        resp_type = b'\1'
+        hi_resp_type = b'\1'
+        reserved1 = b'\0' * 2
+        reserved2 = b'\0' * 4
+        reserved3 = b'\0' * 4
+        reserved4 = b'\0' * 4  # This byte is not in the structure defined in 2.2.2.7 but is in the computation guide, works with it present
+
+        temp = resp_type + hi_resp_type + reserved1 + \
+               reserved2 + \
+               timestamp + \
+               client_challenge + \
+               reserved3 + \
+               target_info.get_data() + reserved4
+
+        return temp
+
+    @staticmethod
+    def _calc_resp(password_hash, server_challenge):
+        """
+        Generate the LM response given a 16-byte password hash and the challenge
+        from the CHALLENGE_MESSAGE
+
+        :param password_hash: A 16-byte password hash
+        :param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
+        :return res: A 24-byte buffer to contain the LM response upon return
+        """
+
+        # padding with zeros to make the hash 21 bytes long
+        password_hash += b'\0' * (21 - len(password_hash))
+
+        res = b''
+        dobj = des.DES(password_hash[0:7])
+        res = res + dobj.encrypt(server_challenge[0:8])
+
+        dobj = des.DES(password_hash[7:14])
+        res = res + dobj.encrypt(server_challenge[0:8])
+
+        dobj = des.DES(password_hash[14:21])
+        res = res + dobj.encrypt(server_challenge[0:8])
+        return res
+
+    @staticmethod
+    def _get_channel_bindings_value(server_certificate_hash):
+        """
+        https://msdn.microsoft.com/en-us/library/windows/desktop/dd919963%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
+        https://blogs.msdn.microsoft.com/openspecification/2013/03/26/ntlm-and-channel-binding-hash-aka-extended-protection-for-authentication/
+
+        Get's the MD5 hash of the gss_channel_bindings_struct to add to the AV_PAIR MSV_AV_CHANNEL_BINDINGS.
+        This method takes in the SHA256 hash (Hash of the DER encoded certificate of the server we are connecting to)
+        and add's it to the gss_channel_bindings_struct. It then gets the MD5 hash and converts this to a
+        byte array in preparation of adding it to the AV_PAIR structure.
+
+        :param server_certificate_hash: The SHA256 hash of the server certificate (DER encoded) NTLM is authenticated to
+        :return channel_bindings: An MD5 hash of the gss_channel_bindings_struct to add to the AV_PAIR MsvChannelBindings
+        """
+        # Channel Binding Tokens support, used for NTLMv2
+        # Decode the SHA256 certificate hash
+        certificate_digest = base64.b16decode(server_certificate_hash)
+
+        # Initialise the GssChannelBindingsStruct and add the certificate_digest to the application_data field
+        gss_channel_bindings = GssChannelBindingsStruct()
+        gss_channel_bindings[gss_channel_bindings.APPLICATION_DATA] = 'tls-server-end-point:'.encode() + certificate_digest
+
+        # Get the gss_channel_bindings_struct and create an MD5 hash
+        channel_bindings_struct_data = gss_channel_bindings.get_data()
+        channel_bindings_hash = hashlib.md5(channel_bindings_struct_data).hexdigest()
+
+        try:
+            cbt_value = bytearray.fromhex(channel_bindings_hash)
+        except TypeError:
+            # Work-around for Python 2.6 bug
+            cbt_value = bytearray.fromhex(unicode(channel_bindings_hash))
+
+        channel_bindings = bytes(cbt_value)
+        return channel_bindings
+
+
+def get_windows_timestamp():
+    # Get Windows Date time, 100 nanoseconds since 1601-01-01 in a 64 bit structure
+    timestamp = struct.pack('<q', (116444736000000000 + calendar.timegm(time.gmtime()) * 10000000))
+
+    return timestamp

packages/wakatime/packages/ntlm_auth/constants.py

@@ -0,0 +1,92 @@
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+"""
+    [MS-NLMP] v28.0 2016-07-14
+
+    2.2 Message Syntax
+    The signature field used in NTLM messages
+"""
+NTLM_SIGNATURE = b'NTLMSSP\0'
+
+"""
+    [MS-NLMP] v28.0 2016-07-14
+
+    2.2 Message Syntax
+    The 3 message type options you can have in a message.
+"""
+class MessageTypes(object):
+    NTLM_NEGOTIATE      = 0x1
+    NTLM_CHALLENGE      = 0x2
+    NTLM_AUTHENTICATE   = 0x3
+
+"""
+    [MS-NLMP] v28.0 2016-07-14
+
+    2.2.2.1 AV_PAIR (MsvAvFlags)
+    A 32-bit value indicated server or client configuration
+"""
+class AvFlags(object):
+    AUTHENTICATION_CONSTRAINED  = 0x1
+    MIC_PROVIDED                = 0x2
+    UNTRUSTED_SPN_SOURCE        = 0x4
+
+"""
+    [MS-NLMP] v28.0 2016-07-14
+
+    2.2.2.5 NEGOTIATE
+    During NTLM authentication, each of the following flags is a possible value of the
+    NegotiateFlags field of the NEGOTIATE_MESSAGE, CHALLENGE_MESSAGE and AUTHENTICATE_MESSAGE,
+    unless otherwise noted. These flags define client or server NTLM capabilities
+    supported by the sender.
+"""
+class NegotiateFlags(object):
+    NTLMSSP_NEGOTIATE_56                            = 0x80000000
+    NTLMSSP_NEGOTIATE_KEY_EXCH                      = 0x40000000
+    NTLMSSP_NEGOTIATE_128                           = 0x20000000
+    NTLMSSP_RESERVED_R1                             = 0x10000000
+    NTLMSSP_RESERVED_R2                             = 0x08000000
+    NTLMSSP_RESERVED_R3                             = 0x04000000
+    NTLMSSP_NEGOTIATE_VERSION                       = 0x02000000
+    NTLMSSP_RESERVED_R4                             = 0x01000000
+    NTLMSSP_NEGOTIATE_TARGET_INFO                   = 0x00800000
+    NTLMSSP_REQUEST_NON_NT_SESSION_KEY              = 0x00400000
+    NTLMSSP_RESERVED_R5                             = 0x00200000
+    NTLMSSP_NEGOTIATE_IDENTITY                      = 0x00100000
+    NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY      = 0x00080000
+    NTLMSSP_RESERVED_R6                             = 0x00040000
+    NTLMSSP_TARGET_TYPE_SERVER                      = 0x00020000
+    NTLMSSP_TARGET_TYPE_DOMAIN                      = 0x00010000
+    NTLMSSP_NEGOTIATE_ALWAYS_SIGN                   = 0x00008000
+    NTLMSSP_RESERVED_R7                             = 0x00004000
+    NTLMSSP_NEGOTIATE_OEM_WORKSTATION_SUPPLIED      = 0x00002000
+    NTLMSSP_NEGOTIATE_OEM_DOMAIN_SUPPLIED           = 0x00001000
+    NTLMSSP_ANOYNMOUS                               = 0x00000800
+    NTLMSSP_RESERVED_R8                             = 0x00000400
+    NTLMSSP_NEGOTIATE_NTLM                          = 0x00000200
+    NTLMSSP_RESERVED_R9                             = 0x00000100
+    NTLMSSP_NEGOTIATE_LM_KEY                        = 0x00000080
+    NTLMSSP_NEGOTIATE_DATAGRAM                      = 0x00000040
+    NTLMSSP_NEGOTIATE_SEAL                          = 0x00000020
+    NTLMSSP_NEGOTIATE_SIGN                          = 0x00000010
+    NTLMSSP_RESERVED_R10                            = 0x00000008
+    NTLMSSP_REQUEST_TARGET                          = 0x00000004
+    NTLMSSP_NEGOTIATE_OEM                           = 0x00000002
+    NTLMSSP_NEGOTIATE_UNICODE                       = 0x00000001
+
+class SignSealConstants(object):
+    # Magic Contants used to get the signing and sealing key for Extended Session Security
+    CLIENT_SIGNING = b"session key to client-to-server signing key magic constant\0"
+    SERVER_SIGNING = b"session key to server-to-client signing key magic constant\0"
+    CLIENT_SEALING = b"session key to client-to-server sealing key magic constant\0"
+    SERVER_SEALING = b"session key to server-to-client sealing key magic constant\0"

packages/wakatime/packages/ntlm_auth/des.py

@@ -0,0 +1,88 @@
+# This file is part of 'NTLM Authorization Proxy Server' http://sourceforge.net/projects/ntlmaps/
+# Copyright 2001 Dmitry A. Rozmanov <dima@xenon.spb.ru>
+#
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+import logging
+import six
+from ntlm_auth import des_c
+
+log = logging.getLogger(__name__)
+
+
+class DES:
+    des_c_obj = None
+
+    def __init__(self, key_str):
+        k = str_to_key56(key_str)
+        k = key56_to_key64(k)
+
+        key_str = b''
+        for i in k:
+            key_str += six.int2byte(i & 0xFF)
+
+        self.des_c_obj = des_c.DES(key_str)
+
+    def encrypt(self, plain_text):
+        return self.des_c_obj.encrypt(plain_text)
+
+    def decrypt(self, crypted_text):
+        return self.des_c_obj.decrypt(crypted_text)
+
+
+DESException = 'DESException'
+
+
+def str_to_key56(key_str):
+
+    if not type(key_str) == six.binary_type:
+        # TODO rsanders high - figure out how to make this not necessary
+        key_str = key_str.encode('ascii')
+
+    if len(key_str) < 7:
+        key_str = key_str + b'\000\000\000\000\000\000\000'[:(7 - len(key_str))]
+    key_56 = []
+    for i in six.iterbytes(key_str[:7]):
+        key_56.append(i)
+
+    return key_56
+
+
+def key56_to_key64(key_56):
+    key = []
+    for i in range(8):
+        key.append(0)
+
+    key[0] = key_56[0]
+    key[1] = ((key_56[0] << 7) & 0xFF) | (key_56[1] >> 1)
+    key[2] = ((key_56[1] << 6) & 0xFF) | (key_56[2] >> 2)
+    key[3] = ((key_56[2] << 5) & 0xFF) | (key_56[3] >> 3)
+    key[4] = ((key_56[3] << 4) & 0xFF) | (key_56[4] >> 4)
+    key[5] = ((key_56[4] << 3) & 0xFF) | (key_56[5] >> 5)
+    key[6] = ((key_56[5] << 2) & 0xFF) | (key_56[6] >> 6)
+    key[7] = (key_56[6] << 1) & 0xFF
+
+    key = set_key_odd_parity(key)
+
+    return key
+
+
+def set_key_odd_parity(key):
+    for i in range(len(key)):
+        for k in range(7):
+            bit = 0
+            t = key[i] >> k
+            bit = (t ^ bit) & 0x1
+        key[i] = (key[i] & 0xFE) | bit
+
+    return key

packages/wakatime/packages/ntlm_auth/des_c.py

@@ -0,0 +1,254 @@
+# This file is part of 'NTLM Authorization Proxy Server' http://sourceforge.net/projects/ntlmaps/
+# Copyright 2001 Dmitry A. Rozmanov <dima@xenon.spb.ru>
+#
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+import six
+
+from ntlm_auth.U32 import U32
+from ntlm_auth.des_data import des_SPtrans, des_skb
+
+def c2l(c):
+    "char[4] to unsigned long"
+    l = U32(c[0])
+    l = l | (U32(c[1]) << 8)
+    l = l | (U32(c[2]) << 16)
+    l = l | (U32(c[3]) << 24)
+    return l
+
+
+def l2c(l):
+    "unsigned long to char[4]"
+    c = []
+    c.append(int(l & U32(0xFF)))
+    c.append(int((l >> 8) & U32(0xFF)))
+    c.append(int((l >> 16) & U32(0xFF)))
+    c.append(int((l >> 24) & U32(0xFF)))
+    return c
+
+
+def D_ENCRYPT(tup, u, t, s):
+    L, R, S = tup
+    # print 'LRS1', L, R, S, u, t, '-->',
+    u = (R ^ s[S])
+    t = R ^ s[S + 1]
+    t = ((t >> 4) + (t << 28))
+    L = L ^ (des_SPtrans[1][int((t) & U32(0x3f))] |
+             des_SPtrans[3][int((t >> 8) & U32(0x3f))] |
+             des_SPtrans[5][int((t >> 16) & U32(0x3f))] |
+             des_SPtrans[7][int((t >> 24) & U32(0x3f))] |
+             des_SPtrans[0][int((u) & U32(0x3f))] |
+             des_SPtrans[2][int((u >> 8) & U32(0x3f))] |
+             des_SPtrans[4][int((u >> 16) & U32(0x3f))] |
+             des_SPtrans[6][int((u >> 24) & U32(0x3f))])
+    # print 'LRS:', L, R, S, u, t
+    return (L, R, S), u, t, s
+
+
+def PERM_OP(tup, n, m):
+    "tup - (a, b, t)"
+    a, b, t = tup
+    t = ((a >> n) ^ b) & m
+    b = b ^ t
+    a = a ^ (t << n)
+    return (a, b, t)
+
+
+def HPERM_OP(tup, n, m):
+    "tup - (a, t)"
+    a, t = tup
+    t = ((a << (16 - n)) ^ a) & m
+    a = a ^ t ^ (t >> (16 - n))
+    return a, t
+
+
+shifts2 = [0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0]
+
+
+class DES:
+    KeySched = None  # des_key_schedule
+
+    def __init__(self, key_str):
+        self.KeySched = des_set_key(key_str)
+
+    def decrypt(self, str):
+        # block - UChar[]
+        block = []
+
+        for i in six.iterbytes(str):
+            block.append(i)
+
+        # print block
+        block = des_ecb_encrypt(block, self.KeySched, 0)
+
+        res = b''
+        for i in block:
+            res = res + six.int2byte(i)
+
+        return res
+
+    def encrypt(self, plaintext):
+        # block - UChar[]
+
+        block = []
+        for i in plaintext:
+            block.append(i)
+
+        block = des_ecb_encrypt(block, self.KeySched, 1)
+
+        res = b''
+
+        for i in block:
+            res += six.int2byte(i)
+
+        return res
+
+
+def des_encript(input, ks, encrypt):
+    # input - U32[]
+    # output - U32[]
+    # ks - des_key_shedule - U32[2][16]
+    # encrypt - int
+    # l, r, t, u - U32
+    # i - int
+    # s - U32[]
+
+    l = input[0]
+    r = input[1]
+    t = U32(0)
+    u = U32(0)
+
+    r, l, t = PERM_OP((r, l, t), 4, U32(0x0f0f0f0f))
+    l, r, t = PERM_OP((l, r, t), 16, U32(0x0000ffff))
+    r, l, t = PERM_OP((r, l, t), 2, U32(0x33333333))
+    l, r, t = PERM_OP((l, r, t), 8, U32(0x00ff00ff))
+    r, l, t = PERM_OP((r, l, t), 1, U32(0x55555555))
+
+    t = (r << 1) | (r >> 31)
+    r = (l << 1) | (l >> 31)
+    l = t
+
+    s = ks  # ???????????????
+    # print l, r
+    if encrypt:
+        for i in range(0, 32, 4):
+            rtup, u, t, s = D_ENCRYPT((l, r, i + 0), u, t, s)
+            l = rtup[0]
+            r = rtup[1]
+            rtup, u, t, s = D_ENCRYPT((r, l, i + 2), u, t, s)
+            r = rtup[0]
+            l = rtup[1]
+    else:
+        for i in range(30, 0, -4):
+            rtup, u, t, s = D_ENCRYPT((l, r, i - 0), u, t, s)
+            l = rtup[0]
+            r = rtup[1]
+            rtup, u, t, s = D_ENCRYPT((r, l, i - 2), u, t, s)
+            r = rtup[0]
+            l = rtup[1]
+        # print l, r
+    l = (l >> 1) | (l << 31)
+    r = (r >> 1) | (r << 31)
+
+    r, l, t = PERM_OP((r, l, t), 1, U32(0x55555555))
+    l, r, t = PERM_OP((l, r, t), 8, U32(0x00ff00ff))
+    r, l, t = PERM_OP((r, l, t), 2, U32(0x33333333))
+    l, r, t = PERM_OP((l, r, t), 16, U32(0x0000ffff))
+    r, l, t = PERM_OP((r, l, t), 4, U32(0x0f0f0f0f))
+
+    output = [l]
+    output.append(r)
+    l, r, t, u = U32(0), U32(0), U32(0), U32(0)
+    return output
+
+
+def des_ecb_encrypt(input, ks, encrypt):
+    # input - des_cblock - UChar[8]
+    # output - des_cblock - UChar[8]
+    # ks - des_key_shedule - U32[2][16]
+    # encrypt - int
+
+    # print input
+    l0 = c2l(input[0:4])
+    l1 = c2l(input[4:8])
+    ll = [l0]
+    ll.append(l1)
+    # print ll
+    ll = des_encript(ll, ks, encrypt)
+    # print ll
+    l0 = ll[0]
+    l1 = ll[1]
+    output = l2c(l0)
+    output = output + l2c(l1)
+    # print output
+    l0, l1, ll[0], ll[1] = U32(0), U32(0), U32(0), U32(0)
+    return output
+
+
+def des_set_key(key):
+    # key - des_cblock - UChar[8]
+    # schedule - des_key_schedule
+
+    # register unsigned long c,d,t,s;
+    # register unsigned char *in;
+    # register unsigned long *k;
+    # register int i;
+
+    # k = schedule
+    # in = key
+
+    k = []
+    c = c2l(key[0:4])
+    d = c2l(key[4:8])
+    t = U32(0)
+
+    d, c, t = PERM_OP((d, c, t), 4, U32(0x0f0f0f0f))
+    c, t = HPERM_OP((c, t), -2, U32(0xcccc0000))
+    d, t = HPERM_OP((d, t), -2, U32(0xcccc0000))
+    d, c, t = PERM_OP((d, c, t), 1, U32(0x55555555))
+    c, d, t = PERM_OP((c, d, t), 8, U32(0x00ff00ff))
+    d, c, t = PERM_OP((d, c, t), 1, U32(0x55555555))
+
+    d = (((d & U32(0x000000ff)) << 16) | (d & U32(0x0000ff00)) | ((d & U32(0x00ff0000)) >> 16) | (
+        (c & U32(0xf0000000)) >> 4))
+    c = c & U32(0x0fffffff)
+
+    for i in range(16):
+        if (shifts2[i]):
+            c = ((c >> 2) | (c << 26))
+            d = ((d >> 2) | (d << 26))
+        else:
+            c = ((c >> 1) | (c << 27))
+            d = ((d >> 1) | (d << 27))
+        c = c & U32(0x0fffffff)
+        d = d & U32(0x0fffffff)
+
+        s = des_skb[0][int((c) & U32(0x3f))] | \
+            des_skb[1][int(((c >> 6) & U32(0x03)) | ((c >> 7) & U32(0x3c)))] | \
+            des_skb[2][int(((c >> 13) & U32(0x0f)) | ((c >> 14) & U32(0x30)))] | \
+            des_skb[3][int(((c >> 20) & U32(0x01)) | ((c >> 21) & U32(0x06)) | ((c >> 22) & U32(0x38)))]
+
+        t = des_skb[4][int((d) & U32(0x3f))] | \
+            des_skb[5][int(((d >> 7) & U32(0x03)) | ((d >> 8) & U32(0x3c)))] | \
+            des_skb[6][int((d >> 15) & U32(0x3f))] | \
+            des_skb[7][int(((d >> 21) & U32(0x0f)) | ((d >> 22) & U32(0x30)))]
+        # print s, t
+
+        k.append(((t << 16) | (s & U32(0x0000ffff))) & U32(0xffffffff))
+        s = ((s >> 16) | (t & U32(0xffff0000)))
+        s = (s << 4) | (s >> 28)
+        k.append(s & U32(0xffffffff))
+
+    schedule = k
+
+    return schedule

packages/wakatime/packages/ntlm_auth/des_data.py

@@ -0,0 +1,348 @@
+# This file is part of 'NTLM Authorization Proxy Server' http://sourceforge.net/projects/ntlmaps/
+# Copyright 2001 Dmitry A. Rozmanov <dima@xenon.spb.ru>
+#
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+from ntlm_auth.U32 import U32
+
+# static unsigned long des_SPtrans[8][64]={
+
+des_SPtrans = \
+    [
+        # nibble 0
+        [
+            U32(0x00820200), U32(0x00020000), U32(0x80800000), U32(0x80820200),
+            U32(0x00800000), U32(0x80020200), U32(0x80020000), U32(0x80800000),
+            U32(0x80020200), U32(0x00820200), U32(0x00820000), U32(0x80000200),
+            U32(0x80800200), U32(0x00800000), U32(0x00000000), U32(0x80020000),
+            U32(0x00020000), U32(0x80000000), U32(0x00800200), U32(0x00020200),
+            U32(0x80820200), U32(0x00820000), U32(0x80000200), U32(0x00800200),
+            U32(0x80000000), U32(0x00000200), U32(0x00020200), U32(0x80820000),
+            U32(0x00000200), U32(0x80800200), U32(0x80820000), U32(0x00000000),
+            U32(0x00000000), U32(0x80820200), U32(0x00800200), U32(0x80020000),
+            U32(0x00820200), U32(0x00020000), U32(0x80000200), U32(0x00800200),
+            U32(0x80820000), U32(0x00000200), U32(0x00020200), U32(0x80800000),
+            U32(0x80020200), U32(0x80000000), U32(0x80800000), U32(0x00820000),
+            U32(0x80820200), U32(0x00020200), U32(0x00820000), U32(0x80800200),
+            U32(0x00800000), U32(0x80000200), U32(0x80020000), U32(0x00000000),
+            U32(0x00020000), U32(0x00800000), U32(0x80800200), U32(0x00820200),
+            U32(0x80000000), U32(0x80820000), U32(0x00000200), U32(0x80020200),
+        ],
+
+        # nibble 1
+        [
+            U32(0x10042004), U32(0x00000000), U32(0x00042000), U32(0x10040000),
+            U32(0x10000004), U32(0x00002004), U32(0x10002000), U32(0x00042000),
+            U32(0x00002000), U32(0x10040004), U32(0x00000004), U32(0x10002000),
+            U32(0x00040004), U32(0x10042000), U32(0x10040000), U32(0x00000004),
+            U32(0x00040000), U32(0x10002004), U32(0x10040004), U32(0x00002000),
+            U32(0x00042004), U32(0x10000000), U32(0x00000000), U32(0x00040004),
+            U32(0x10002004), U32(0x00042004), U32(0x10042000), U32(0x10000004),
+            U32(0x10000000), U32(0x00040000), U32(0x00002004), U32(0x10042004),
+            U32(0x00040004), U32(0x10042000), U32(0x10002000), U32(0x00042004),
+            U32(0x10042004), U32(0x00040004), U32(0x10000004), U32(0x00000000),
+            U32(0x10000000), U32(0x00002004), U32(0x00040000), U32(0x10040004),
+            U32(0x00002000), U32(0x10000000), U32(0x00042004), U32(0x10002004),
+            U32(0x10042000), U32(0x00002000), U32(0x00000000), U32(0x10000004),
+            U32(0x00000004), U32(0x10042004), U32(0x00042000), U32(0x10040000),
+            U32(0x10040004), U32(0x00040000), U32(0x00002004), U32(0x10002000),
+            U32(0x10002004), U32(0x00000004), U32(0x10040000), U32(0x00042000),
+        ],
+
+        # nibble 2
+        [
+            U32(0x41000000), U32(0x01010040), U32(0x00000040), U32(0x41000040),
+            U32(0x40010000), U32(0x01000000), U32(0x41000040), U32(0x00010040),
+            U32(0x01000040), U32(0x00010000), U32(0x01010000), U32(0x40000000),
+            U32(0x41010040), U32(0x40000040), U32(0x40000000), U32(0x41010000),
+            U32(0x00000000), U32(0x40010000), U32(0x01010040), U32(0x00000040),
+            U32(0x40000040), U32(0x41010040), U32(0x00010000), U32(0x41000000),
+            U32(0x41010000), U32(0x01000040), U32(0x40010040), U32(0x01010000),
+            U32(0x00010040), U32(0x00000000), U32(0x01000000), U32(0x40010040),
+            U32(0x01010040), U32(0x00000040), U32(0x40000000), U32(0x00010000),
+            U32(0x40000040), U32(0x40010000), U32(0x01010000), U32(0x41000040),
+            U32(0x00000000), U32(0x01010040), U32(0x00010040), U32(0x41010000),
+            U32(0x40010000), U32(0x01000000), U32(0x41010040), U32(0x40000000),
+            U32(0x40010040), U32(0x41000000), U32(0x01000000), U32(0x41010040),
+            U32(0x00010000), U32(0x01000040), U32(0x41000040), U32(0x00010040),
+            U32(0x01000040), U32(0x00000000), U32(0x41010000), U32(0x40000040),
+            U32(0x41000000), U32(0x40010040), U32(0x00000040), U32(0x01010000),
+        ],
+
+        # nibble 3
+        [
+            U32(0x00100402), U32(0x04000400), U32(0x00000002), U32(0x04100402),
+            U32(0x00000000), U32(0x04100000), U32(0x04000402), U32(0x00100002),
+            U32(0x04100400), U32(0x04000002), U32(0x04000000), U32(0x00000402),
+            U32(0x04000002), U32(0x00100402), U32(0x00100000), U32(0x04000000),
+            U32(0x04100002), U32(0x00100400), U32(0x00000400), U32(0x00000002),
+            U32(0x00100400), U32(0x04000402), U32(0x04100000), U32(0x00000400),
+            U32(0x00000402), U32(0x00000000), U32(0x00100002), U32(0x04100400),
+            U32(0x04000400), U32(0x04100002), U32(0x04100402), U32(0x00100000),
+            U32(0x04100002), U32(0x00000402), U32(0x00100000), U32(0x04000002),
+            U32(0x00100400), U32(0x04000400), U32(0x00000002), U32(0x04100000),
+            U32(0x04000402), U32(0x00000000), U32(0x00000400), U32(0x00100002),
+            U32(0x00000000), U32(0x04100002), U32(0x04100400), U32(0x00000400),
+            U32(0x04000000), U32(0x04100402), U32(0x00100402), U32(0x00100000),
+            U32(0x04100402), U32(0x00000002), U32(0x04000400), U32(0x00100402),
+            U32(0x00100002), U32(0x00100400), U32(0x04100000), U32(0x04000402),
+            U32(0x00000402), U32(0x04000000), U32(0x04000002), U32(0x04100400),
+        ],
+
+        # nibble 4
+        [
+            U32(0x02000000), U32(0x00004000), U32(0x00000100), U32(0x02004108),
+            U32(0x02004008), U32(0x02000100), U32(0x00004108), U32(0x02004000),
+            U32(0x00004000), U32(0x00000008), U32(0x02000008), U32(0x00004100),
+            U32(0x02000108), U32(0x02004008), U32(0x02004100), U32(0x00000000),
+            U32(0x00004100), U32(0x02000000), U32(0x00004008), U32(0x00000108),
+            U32(0x02000100), U32(0x00004108), U32(0x00000000), U32(0x02000008),
+            U32(0x00000008), U32(0x02000108), U32(0x02004108), U32(0x00004008),
+            U32(0x02004000), U32(0x00000100), U32(0x00000108), U32(0x02004100),
+            U32(0x02004100), U32(0x02000108), U32(0x00004008), U32(0x02004000),
+            U32(0x00004000), U32(0x00000008), U32(0x02000008), U32(0x02000100),
+            U32(0x02000000), U32(0x00004100), U32(0x02004108), U32(0x00000000),
+            U32(0x00004108), U32(0x02000000), U32(0x00000100), U32(0x00004008),
+            U32(0x02000108), U32(0x00000100), U32(0x00000000), U32(0x02004108),
+            U32(0x02004008), U32(0x02004100), U32(0x00000108), U32(0x00004000),
+            U32(0x00004100), U32(0x02004008), U32(0x02000100), U32(0x00000108),
+            U32(0x00000008), U32(0x00004108), U32(0x02004000), U32(0x02000008),
+        ],
+
+        # nibble 5
+        [
+            U32(0x20000010), U32(0x00080010), U32(0x00000000), U32(0x20080800),
+            U32(0x00080010), U32(0x00000800), U32(0x20000810), U32(0x00080000),
+            U32(0x00000810), U32(0x20080810), U32(0x00080800), U32(0x20000000),
+            U32(0x20000800), U32(0x20000010), U32(0x20080000), U32(0x00080810),
+            U32(0x00080000), U32(0x20000810), U32(0x20080010), U32(0x00000000),
+            U32(0x00000800), U32(0x00000010), U32(0x20080800), U32(0x20080010),
+            U32(0x20080810), U32(0x20080000), U32(0x20000000), U32(0x00000810),
+            U32(0x00000010), U32(0x00080800), U32(0x00080810), U32(0x20000800),
+            U32(0x00000810), U32(0x20000000), U32(0x20000800), U32(0x00080810),
+            U32(0x20080800), U32(0x00080010), U32(0x00000000), U32(0x20000800),
+            U32(0x20000000), U32(0x00000800), U32(0x20080010), U32(0x00080000),
+            U32(0x00080010), U32(0x20080810), U32(0x00080800), U32(0x00000010),
+            U32(0x20080810), U32(0x00080800), U32(0x00080000), U32(0x20000810),
+            U32(0x20000010), U32(0x20080000), U32(0x00080810), U32(0x00000000),
+            U32(0x00000800), U32(0x20000010), U32(0x20000810), U32(0x20080800),
+            U32(0x20080000), U32(0x00000810), U32(0x00000010), U32(0x20080010),
+        ],
+
+        # nibble 6
+        [
+            U32(0x00001000), U32(0x00000080), U32(0x00400080), U32(0x00400001),
+            U32(0x00401081), U32(0x00001001), U32(0x00001080), U32(0x00000000),
+            U32(0x00400000), U32(0x00400081), U32(0x00000081), U32(0x00401000),
+            U32(0x00000001), U32(0x00401080), U32(0x00401000), U32(0x00000081),
+            U32(0x00400081), U32(0x00001000), U32(0x00001001), U32(0x00401081),
+            U32(0x00000000), U32(0x00400080), U32(0x00400001), U32(0x00001080),
+            U32(0x00401001), U32(0x00001081), U32(0x00401080), U32(0x00000001),
+            U32(0x00001081), U32(0x00401001), U32(0x00000080), U32(0x00400000),
+            U32(0x00001081), U32(0x00401000), U32(0x00401001), U32(0x00000081),
+            U32(0x00001000), U32(0x00000080), U32(0x00400000), U32(0x00401001),
+            U32(0x00400081), U32(0x00001081), U32(0x00001080), U32(0x00000000),
+            U32(0x00000080), U32(0x00400001), U32(0x00000001), U32(0x00400080),
+            U32(0x00000000), U32(0x00400081), U32(0x00400080), U32(0x00001080),
+            U32(0x00000081), U32(0x00001000), U32(0x00401081), U32(0x00400000),
+            U32(0x00401080), U32(0x00000001), U32(0x00001001), U32(0x00401081),
+            U32(0x00400001), U32(0x00401080), U32(0x00401000), U32(0x00001001),
+        ],
+
+        # nibble 7
+        [
+            U32(0x08200020), U32(0x08208000), U32(0x00008020), U32(0x00000000),
+            U32(0x08008000), U32(0x00200020), U32(0x08200000), U32(0x08208020),
+            U32(0x00000020), U32(0x08000000), U32(0x00208000), U32(0x00008020),
+            U32(0x00208020), U32(0x08008020), U32(0x08000020), U32(0x08200000),
+            U32(0x00008000), U32(0x00208020), U32(0x00200020), U32(0x08008000),
+            U32(0x08208020), U32(0x08000020), U32(0x00000000), U32(0x00208000),
+            U32(0x08000000), U32(0x00200000), U32(0x08008020), U32(0x08200020),
+            U32(0x00200000), U32(0x00008000), U32(0x08208000), U32(0x00000020),
+            U32(0x00200000), U32(0x00008000), U32(0x08000020), U32(0x08208020),
+            U32(0x00008020), U32(0x08000000), U32(0x00000000), U32(0x00208000),
+            U32(0x08200020), U32(0x08008020), U32(0x08008000), U32(0x00200020),
+            U32(0x08208000), U32(0x00000020), U32(0x00200020), U32(0x08008000),
+            U32(0x08208020), U32(0x00200000), U32(0x08200000), U32(0x08000020),
+            U32(0x00208000), U32(0x00008020), U32(0x08008020), U32(0x08200000),
+            U32(0x00000020), U32(0x08208000), U32(0x00208020), U32(0x00000000),
+            U32(0x08000000), U32(0x08200020), U32(0x00008000), U32(0x00208020),
+        ],
+    ]
+
+# static unsigned long des_skb[8][64]={
+
+des_skb = \
+    [
+        # for C bits (numbered as per FIPS 46) 1 2 3 4 5 6
+        [
+            U32(0x00000000), U32(0x00000010), U32(0x20000000), U32(0x20000010),
+            U32(0x00010000), U32(0x00010010), U32(0x20010000), U32(0x20010010),
+            U32(0x00000800), U32(0x00000810), U32(0x20000800), U32(0x20000810),
+            U32(0x00010800), U32(0x00010810), U32(0x20010800), U32(0x20010810),
+            U32(0x00000020), U32(0x00000030), U32(0x20000020), U32(0x20000030),
+            U32(0x00010020), U32(0x00010030), U32(0x20010020), U32(0x20010030),
+            U32(0x00000820), U32(0x00000830), U32(0x20000820), U32(0x20000830),
+            U32(0x00010820), U32(0x00010830), U32(0x20010820), U32(0x20010830),
+            U32(0x00080000), U32(0x00080010), U32(0x20080000), U32(0x20080010),
+            U32(0x00090000), U32(0x00090010), U32(0x20090000), U32(0x20090010),
+            U32(0x00080800), U32(0x00080810), U32(0x20080800), U32(0x20080810),
+            U32(0x00090800), U32(0x00090810), U32(0x20090800), U32(0x20090810),
+            U32(0x00080020), U32(0x00080030), U32(0x20080020), U32(0x20080030),
+            U32(0x00090020), U32(0x00090030), U32(0x20090020), U32(0x20090030),
+            U32(0x00080820), U32(0x00080830), U32(0x20080820), U32(0x20080830),
+            U32(0x00090820), U32(0x00090830), U32(0x20090820), U32(0x20090830),
+        ],
+
+        # for C bits (numbered as per FIPS 46) 7 8 10 11 12 13
+        [
+            U32(0x00000000), U32(0x02000000), U32(0x00002000), U32(0x02002000),
+            U32(0x00200000), U32(0x02200000), U32(0x00202000), U32(0x02202000),
+            U32(0x00000004), U32(0x02000004), U32(0x00002004), U32(0x02002004),
+            U32(0x00200004), U32(0x02200004), U32(0x00202004), U32(0x02202004),
+            U32(0x00000400), U32(0x02000400), U32(0x00002400), U32(0x02002400),
+            U32(0x00200400), U32(0x02200400), U32(0x00202400), U32(0x02202400),
+            U32(0x00000404), U32(0x02000404), U32(0x00002404), U32(0x02002404),
+            U32(0x00200404), U32(0x02200404), U32(0x00202404), U32(0x02202404),
+            U32(0x10000000), U32(0x12000000), U32(0x10002000), U32(0x12002000),
+            U32(0x10200000), U32(0x12200000), U32(0x10202000), U32(0x12202000),
+            U32(0x10000004), U32(0x12000004), U32(0x10002004), U32(0x12002004),
+            U32(0x10200004), U32(0x12200004), U32(0x10202004), U32(0x12202004),
+            U32(0x10000400), U32(0x12000400), U32(0x10002400), U32(0x12002400),
+            U32(0x10200400), U32(0x12200400), U32(0x10202400), U32(0x12202400),
+            U32(0x10000404), U32(0x12000404), U32(0x10002404), U32(0x12002404),
+            U32(0x10200404), U32(0x12200404), U32(0x10202404), U32(0x12202404),
+        ],
+
+        # for C bits (numbered as per FIPS 46) 14 15 16 17 19 20
+        [
+            U32(0x00000000), U32(0x00000001), U32(0x00040000), U32(0x00040001),
+            U32(0x01000000), U32(0x01000001), U32(0x01040000), U32(0x01040001),
+            U32(0x00000002), U32(0x00000003), U32(0x00040002), U32(0x00040003),
+            U32(0x01000002), U32(0x01000003), U32(0x01040002), U32(0x01040003),
+            U32(0x00000200), U32(0x00000201), U32(0x00040200), U32(0x00040201),
+            U32(0x01000200), U32(0x01000201), U32(0x01040200), U32(0x01040201),
+            U32(0x00000202), U32(0x00000203), U32(0x00040202), U32(0x00040203),
+            U32(0x01000202), U32(0x01000203), U32(0x01040202), U32(0x01040203),
+            U32(0x08000000), U32(0x08000001), U32(0x08040000), U32(0x08040001),
+            U32(0x09000000), U32(0x09000001), U32(0x09040000), U32(0x09040001),
+            U32(0x08000002), U32(0x08000003), U32(0x08040002), U32(0x08040003),
+            U32(0x09000002), U32(0x09000003), U32(0x09040002), U32(0x09040003),
+            U32(0x08000200), U32(0x08000201), U32(0x08040200), U32(0x08040201),
+            U32(0x09000200), U32(0x09000201), U32(0x09040200), U32(0x09040201),
+            U32(0x08000202), U32(0x08000203), U32(0x08040202), U32(0x08040203),
+            U32(0x09000202), U32(0x09000203), U32(0x09040202), U32(0x09040203),
+        ],
+
+        # for C bits (numbered as per FIPS 46) 21 23 24 26 27 28
+        [
+            U32(0x00000000), U32(0x00100000), U32(0x00000100), U32(0x00100100),
+            U32(0x00000008), U32(0x00100008), U32(0x00000108), U32(0x00100108),
+            U32(0x00001000), U32(0x00101000), U32(0x00001100), U32(0x00101100),
+            U32(0x00001008), U32(0x00101008), U32(0x00001108), U32(0x00101108),
+            U32(0x04000000), U32(0x04100000), U32(0x04000100), U32(0x04100100),
+            U32(0x04000008), U32(0x04100008), U32(0x04000108), U32(0x04100108),
+            U32(0x04001000), U32(0x04101000), U32(0x04001100), U32(0x04101100),
+            U32(0x04001008), U32(0x04101008), U32(0x04001108), U32(0x04101108),
+            U32(0x00020000), U32(0x00120000), U32(0x00020100), U32(0x00120100),
+            U32(0x00020008), U32(0x00120008), U32(0x00020108), U32(0x00120108),
+            U32(0x00021000), U32(0x00121000), U32(0x00021100), U32(0x00121100),
+            U32(0x00021008), U32(0x00121008), U32(0x00021108), U32(0x00121108),
+            U32(0x04020000), U32(0x04120000), U32(0x04020100), U32(0x04120100),
+            U32(0x04020008), U32(0x04120008), U32(0x04020108), U32(0x04120108),
+            U32(0x04021000), U32(0x04121000), U32(0x04021100), U32(0x04121100),
+            U32(0x04021008), U32(0x04121008), U32(0x04021108), U32(0x04121108),
+        ],
+
+        # for D bits (numbered as per FIPS 46) 1 2 3 4 5 6
+        [
+            U32(0x00000000), U32(0x10000000), U32(0x00010000), U32(0x10010000),
+            U32(0x00000004), U32(0x10000004), U32(0x00010004), U32(0x10010004),
+            U32(0x20000000), U32(0x30000000), U32(0x20010000), U32(0x30010000),
+            U32(0x20000004), U32(0x30000004), U32(0x20010004), U32(0x30010004),
+            U32(0x00100000), U32(0x10100000), U32(0x00110000), U32(0x10110000),
+            U32(0x00100004), U32(0x10100004), U32(0x00110004), U32(0x10110004),
+            U32(0x20100000), U32(0x30100000), U32(0x20110000), U32(0x30110000),
+            U32(0x20100004), U32(0x30100004), U32(0x20110004), U32(0x30110004),
+            U32(0x00001000), U32(0x10001000), U32(0x00011000), U32(0x10011000),
+            U32(0x00001004), U32(0x10001004), U32(0x00011004), U32(0x10011004),
+            U32(0x20001000), U32(0x30001000), U32(0x20011000), U32(0x30011000),
+            U32(0x20001004), U32(0x30001004), U32(0x20011004), U32(0x30011004),
+            U32(0x00101000), U32(0x10101000), U32(0x00111000), U32(0x10111000),
+            U32(0x00101004), U32(0x10101004), U32(0x00111004), U32(0x10111004),
+            U32(0x20101000), U32(0x30101000), U32(0x20111000), U32(0x30111000),
+            U32(0x20101004), U32(0x30101004), U32(0x20111004), U32(0x30111004),
+        ],
+
+        # for D bits (numbered as per FIPS 46) 8 9 11 12 13 14
+        [
+            U32(0x00000000), U32(0x08000000), U32(0x00000008), U32(0x08000008),
+            U32(0x00000400), U32(0x08000400), U32(0x00000408), U32(0x08000408),
+            U32(0x00020000), U32(0x08020000), U32(0x00020008), U32(0x08020008),
+            U32(0x00020400), U32(0x08020400), U32(0x00020408), U32(0x08020408),
+            U32(0x00000001), U32(0x08000001), U32(0x00000009), U32(0x08000009),
+            U32(0x00000401), U32(0x08000401), U32(0x00000409), U32(0x08000409),
+            U32(0x00020001), U32(0x08020001), U32(0x00020009), U32(0x08020009),
+            U32(0x00020401), U32(0x08020401), U32(0x00020409), U32(0x08020409),
+            U32(0x02000000), U32(0x0A000000), U32(0x02000008), U32(0x0A000008),
+            U32(0x02000400), U32(0x0A000400), U32(0x02000408), U32(0x0A000408),
+            U32(0x02020000), U32(0x0A020000), U32(0x02020008), U32(0x0A020008),
+            U32(0x02020400), U32(0x0A020400), U32(0x02020408), U32(0x0A020408),
+            U32(0x02000001), U32(0x0A000001), U32(0x02000009), U32(0x0A000009),
+            U32(0x02000401), U32(0x0A000401), U32(0x02000409), U32(0x0A000409),
+            U32(0x02020001), U32(0x0A020001), U32(0x02020009), U32(0x0A020009),
+            U32(0x02020401), U32(0x0A020401), U32(0x02020409), U32(0x0A020409),
+        ],
+
+        # for D bits (numbered as per FIPS 46) 16 17 18 19 20 21
+        [
+            U32(0x00000000), U32(0x00000100), U32(0x00080000), U32(0x00080100),
+            U32(0x01000000), U32(0x01000100), U32(0x01080000), U32(0x01080100),
+            U32(0x00000010), U32(0x00000110), U32(0x00080010), U32(0x00080110),
+            U32(0x01000010), U32(0x01000110), U32(0x01080010), U32(0x01080110),
+            U32(0x00200000), U32(0x00200100), U32(0x00280000), U32(0x00280100),
+            U32(0x01200000), U32(0x01200100), U32(0x01280000), U32(0x01280100),
+            U32(0x00200010), U32(0x00200110), U32(0x00280010), U32(0x00280110),
+            U32(0x01200010), U32(0x01200110), U32(0x01280010), U32(0x01280110),
+            U32(0x00000200), U32(0x00000300), U32(0x00080200), U32(0x00080300),
+            U32(0x01000200), U32(0x01000300), U32(0x01080200), U32(0x01080300),
+            U32(0x00000210), U32(0x00000310), U32(0x00080210), U32(0x00080310),
+            U32(0x01000210), U32(0x01000310), U32(0x01080210), U32(0x01080310),
+            U32(0x00200200), U32(0x00200300), U32(0x00280200), U32(0x00280300),
+            U32(0x01200200), U32(0x01200300), U32(0x01280200), U32(0x01280300),
+            U32(0x00200210), U32(0x00200310), U32(0x00280210), U32(0x00280310),
+            U32(0x01200210), U32(0x01200310), U32(0x01280210), U32(0x01280310),
+        ],
+
+        # for D bits (numbered as per FIPS 46) 22 23 24 25 27 28
+        [
+            U32(0x00000000), U32(0x04000000), U32(0x00040000), U32(0x04040000),
+            U32(0x00000002), U32(0x04000002), U32(0x00040002), U32(0x04040002),
+            U32(0x00002000), U32(0x04002000), U32(0x00042000), U32(0x04042000),
+            U32(0x00002002), U32(0x04002002), U32(0x00042002), U32(0x04042002),
+            U32(0x00000020), U32(0x04000020), U32(0x00040020), U32(0x04040020),
+            U32(0x00000022), U32(0x04000022), U32(0x00040022), U32(0x04040022),
+            U32(0x00002020), U32(0x04002020), U32(0x00042020), U32(0x04042020),
+            U32(0x00002022), U32(0x04002022), U32(0x00042022), U32(0x04042022),
+            U32(0x00000800), U32(0x04000800), U32(0x00040800), U32(0x04040800),
+            U32(0x00000802), U32(0x04000802), U32(0x00040802), U32(0x04040802),
+            U32(0x00002800), U32(0x04002800), U32(0x00042800), U32(0x04042800),
+            U32(0x00002802), U32(0x04002802), U32(0x00042802), U32(0x04042802),
+            U32(0x00000820), U32(0x04000820), U32(0x00040820), U32(0x04040820),
+            U32(0x00000822), U32(0x04000822), U32(0x00040822), U32(0x04040822),
+            U32(0x00002820), U32(0x04002820), U32(0x00042820), U32(0x04042820),
+            U32(0x00002822), U32(0x04002822), U32(0x00042822), U32(0x04042822),
+        ]
+
+    ]

packages/wakatime/packages/ntlm_auth/gss_channel_bindings.py

@@ -0,0 +1,67 @@
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+import struct
+
+"""
+    This is not the easiest structure to understand, ultimately this is a set structure
+    as defined by Microsoft. Channel Binding Tokens set the SHA256 hash of the server
+    certificate to the application_data field and then ultimately creates the MD5 hash
+    to include in the NTLM auth from there. This class is just designed to create the
+    bindings structure which is then used by compute_response.py to do the rest of the
+    work.
+
+    For more infor on how this works and how it is derived, this is a great link;
+    https://blogs.msdn.microsoft.com/openspecification/2013/03/26/ntlm-and-channel-binding-hash-aka-extended-protection-for-authentication/
+"""
+class GssChannelBindingsStruct(object):
+    INITIATOR_ADDTYPE = 'initiator_addtype'
+    INITIATOR_ADDRESS_LENGTH = 'initiator_address_length'
+    ACCEPTOR_ADDRTYPE = 'acceptor_addrtype'
+    ACCEPTOR_ADDRESS_LENGTH = 'acceptor_address_length'
+    APPLICATION_DATA_LENGTH = 'application_data_length'
+    INITIATOR_ADDRESS = 'initiator_address'
+    ACCEPTOR_ADDRESS = 'acceptor_address'
+    APPLICATION_DATA = 'application_data'
+
+    def __init__(self):
+        self.fields = {}
+        self.fields[self.INITIATOR_ADDTYPE] = 0
+        self.fields[self.INITIATOR_ADDRESS_LENGTH] = 0
+        self.fields[self.ACCEPTOR_ADDRTYPE] = 0
+        self.fields[self.ACCEPTOR_ADDRESS_LENGTH] = 0
+        self.fields[self.APPLICATION_DATA_LENGTH] = 0
+        self.fields[self.INITIATOR_ADDRESS] = b''
+        self.fields[self.ACCEPTOR_ADDRESS] = b''
+        self.fields[self.APPLICATION_DATA] = b''
+
+    def __setitem__(self, key, value):
+        self.fields[key] = value
+
+    def get_data(self):
+        # Set the lengths of each len field in case they have changed
+        self.fields[self.INITIATOR_ADDRESS_LENGTH] = len(self.fields[self.INITIATOR_ADDRESS])
+        self.fields[self.ACCEPTOR_ADDRESS_LENGTH] = len(self.fields[self.ACCEPTOR_ADDRESS])
+        self.fields[self.APPLICATION_DATA_LENGTH] = len(self.fields[self.APPLICATION_DATA])
+
+        # Add all the values together to create the gss_channel_bindings_struct
+        data = struct.pack('<L', self.fields[self.INITIATOR_ADDTYPE]) + \
+               struct.pack('<L', self.fields[self.INITIATOR_ADDRESS_LENGTH]) + \
+               self.fields[self.INITIATOR_ADDRESS] + \
+               struct.pack('<L', self.fields[self.ACCEPTOR_ADDRTYPE]) + \
+               struct.pack('<L', self.fields[self.ACCEPTOR_ADDRESS_LENGTH]) + \
+               self.fields[self.ACCEPTOR_ADDRESS] + \
+               struct.pack('<L', self.fields[self.APPLICATION_DATA_LENGTH]) + \
+               self.fields[self.APPLICATION_DATA]
+
+        return data

packages/wakatime/packages/ntlm_auth/messages.py

@@ -0,0 +1,359 @@
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+import hmac
+import os
+import struct
+from ntlm_auth.compute_response import ComputeResponse
+from ntlm_auth.constants import NegotiateFlags, MessageTypes, NTLM_SIGNATURE, AvFlags
+from ntlm_auth.rc4 import ARC4
+from ntlm_auth.target_info import TargetInfo
+
+class NegotiateMessage(object):
+    EXPECTED_BODY_LENGTH = 40
+
+    """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.1.1 NEGOTIATE_MESSAGE
+        The NEGOTIATE_MESSAGE defines an NTLM Negotiate message that is sent from the client to
+        the server. This message allows the client to specify its supported NTLM options to
+        the server.
+
+        :param negotiate_flags: A NEGOTIATE structure that contains a set of bit flags. These flags are the options the client supports
+        :param domain_name: The domain name of the user to authenticate with, default is None
+        :param workstation: The worksation of the client machine, default is None
+
+        Attributes:
+            signature: An 8-byte character array that MUST contain the ASCII string 'NTLMSSP\0'
+            message_type: A 32-bit unsigned integer that indicates the message type. This field must be set to 0x00000001
+            negotiate_flags: A NEGOTIATE structure that contains a set of bit flags. These flags are the options the client supports
+            version: Contains the windows version info of the client. It is used only debugging purposes and are only set when NTLMSSP_NEGOTIATE_VERSION flag is set
+            domain_name: A byte-array that contains the name of the client authentication domain that MUST Be encoded in the negotiated character set
+            workstation: A byte-array that contains the name of the client machine that MUST Be encoded in the negotiated character set
+    """
+    def __init__(self, negotiate_flags, domain_name, workstation):
+        self.signature = NTLM_SIGNATURE
+        self.message_type = struct.pack('<L', MessageTypes.NTLM_NEGOTIATE)
+
+        # Check if the domain_name value is set, if it is, make sure the negotiate_flag is also set
+        if domain_name is None:
+            self.domain_name = ''
+        else:
+            self.domain_name = domain_name
+            negotiate_flags |= NegotiateFlags.NTLMSSP_NEGOTIATE_OEM_DOMAIN_SUPPLIED
+
+        # Check if the workstation value is set, if it is, make sure the negotiate_flag is also set
+        if workstation is None:
+            self.workstation = ''
+        else:
+            self.workstation = workstation
+            negotiate_flags |= NegotiateFlags.NTLMSSP_NEGOTIATE_OEM_WORKSTATION_SUPPLIED
+
+        # Set the encoding flag to use OEM, remove UNICODE if set as it isn't support in this message
+        negotiate_flags -= NegotiateFlags.NTLMSSP_NEGOTIATE_UNICODE
+        negotiate_flags |= NegotiateFlags.NTLMSSP_NEGOTIATE_OEM
+        self.domain_name = self.domain_name.encode('ascii')
+        self.workstation = self.workstation.encode('ascii')
+
+        self.version = get_version(negotiate_flags)
+
+        self.negotiate_flags = struct.pack('<I', negotiate_flags)
+
+    def get_data(self):
+        payload_offset = self.EXPECTED_BODY_LENGTH
+
+        # DomainNameFields - 8 bytes
+        domain_name_len = struct.pack('<H', len(self.domain_name))
+        domain_name_max_len = struct.pack('<H', len(self.domain_name))
+        domain_name_buffer_offset = struct.pack('<I', payload_offset)
+        payload_offset += len(self.domain_name)
+
+        # WorkstationFields - 8 bytes
+        workstation_len = struct.pack('<H', len(self.workstation))
+        workstation_max_len = struct.pack('<H', len(self.workstation))
+        workstation_buffer_offset = struct.pack('<I', payload_offset)
+        payload_offset += len(self.workstation)
+
+        # Payload - variable length
+        payload = self.domain_name
+        payload += self.workstation
+
+        # Bring the header values together into 1 message
+        msg1 = self.signature
+        msg1 += self.message_type
+        msg1 += self.negotiate_flags
+        msg1 += domain_name_len
+        msg1 += domain_name_max_len
+        msg1 += domain_name_buffer_offset
+        msg1 += workstation_len
+        msg1 += workstation_max_len
+        msg1 += workstation_buffer_offset
+        msg1 += self.version
+
+        assert self.EXPECTED_BODY_LENGTH == len(msg1), "BODY_LENGTH: %d != msg1: %d" % (self.EXPECTED_BODY_LENGTH, len(msg1))
+
+        # Adding the payload data to the message
+        msg1 += payload
+        return msg1
+
+class ChallengeMessage(object):
+    """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.1.2 CHALLENGE_MESSAGE
+        The CHALLENGE_MESSAGE defines an NTLM challenge message that is sent from the server to
+        the client. The CHALLENGE_MESSAGE is used by the server to challenge the client to prove
+        its identity, For connection-oriented requests, the CHALLENGE_MESSAGE generated by the
+        server is in response to the NEGOTIATE_MESSAGE from the client.
+
+        :param msg2: The CHALLENGE_MESSAGE received from the server after sending our NEGOTIATE_MESSAGE. This has
+                        been decoded from a base64 string
+
+        Attributes
+            signature: An 8-byte character array that MUST contain the ASCII string 'NTLMSSP\0'
+            message_type: A 32-bit unsigned integer that indicates the message type. This field must be set to 0x00000002
+            negotiate_flags: A NEGOTIATE strucutre that contains a set of bit flags. The server sets flags to indicate options it supports
+            server_challenge: A 64-bit value that contains the NTLM challenge. The challenge is a 64-bit nonce. Used in the AuthenticateMessage message
+            reserved: An 8-byte array whose elements MUST be zero when sent and MUST be ignored on receipt
+            version: When NTLMSSP_NEGOTIATE_VERSION flag is set in negotiate_flags field which contains the windows version info. Used only for debugging purposes
+            target_name: When NTLMSSP_REQUEST_TARGET is set is a byte array that contains the name of the server authentication realm. In a domain environment this is the domain name not server name
+            target_info: When NTLMSSP_NEGOTIATE_TARGET_INFO is set is a byte array that contains a sequence of AV_PAIR structures (target_info.py)
+    """
+    def __init__(self, msg2):
+        self.data = msg2
+        # Setting the object values from the raw_challenge_message
+        self.signature = msg2[0:8]
+        self.message_type = struct.unpack("<I", msg2[8:12])[0]
+        self.negotiate_flags = struct.unpack("<I", msg2[20:24])[0]
+        self.server_challenge = msg2[24:32]
+        self.reserved = msg2[32:40]
+
+        if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_VERSION and self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY:
+            size = len(msg2)
+            self.version = struct.unpack("<q", msg2[48:56])[0]
+        else:
+            self.version = None
+
+        if self.negotiate_flags & NegotiateFlags.NTLMSSP_REQUEST_TARGET:
+            target_name_len = struct.unpack("<H", msg2[12:14])[0]
+            target_name_max_len = struct.unpack("<H", msg2[14:16])[0]
+            target_name_buffer_offset = struct.unpack("<I", msg2[16:20])[0]
+            self.target_name = msg2[target_name_buffer_offset:target_name_buffer_offset + target_name_len]
+        else:
+            self.target_name = None
+
+        if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_TARGET_INFO:
+            target_info_len = struct.unpack("<H", msg2[40:42])[0]
+            target_info_max_len = struct.unpack("<H", msg2[42:44])[0]
+            target_info_buffer_offset = struct.unpack("<I", msg2[44:48])[0]
+
+            target_info_raw = msg2[target_info_buffer_offset:target_info_buffer_offset + target_info_len]
+            self.target_info = TargetInfo(target_info_raw)
+        else:
+            self.target_info = None
+
+        # Verify initial integrity of the message, it matches what should be there
+        assert self.signature == NTLM_SIGNATURE
+        assert self.message_type == MessageTypes.NTLM_CHALLENGE
+
+    def get_data(self):
+        return self.data
+
+class AuthenticateMessage(object):
+    EXPECTED_BODY_LENGTH = 72
+    EXPECTED_BODY_LENGTH_WITH_MIC = 88
+
+    """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.1.3 AUTHENTICATE_MESSAGE
+        The AUTHENTICATE_MESSAGE defines an NTLM authenticate message that is sent from the
+        client to the server after the CHALLENGE_MESSAGE is processed by the client.
+
+        :param user_name: The user name of the user we are trying to authenticate with
+        :param password: The password of the user we are trying to authenticate with
+        :param domain_name: The domain name of the user account we are authenticated with, default is None
+        :param workstation: The workstation we are using to authenticate with, default is None
+        :param challenge_message: A ChallengeMessage object that was received from the server after the negotiate_message
+        :param ntlm_compatibility: The Lan Manager Compatibility Level, used to determine what NTLM auth version to use, see Ntlm in ntlm.py for more details
+        :param server_certificate_hash: The SHA256 hash string of the server certificate (DER encoded) NTLM is authenticating to. This is used to add
+                                          to the gss_channel_bindings_struct for Channel Binding Tokens support. If none is passed through then ntlm-auth
+                                          will not use Channel Binding Tokens when authenticating with the server which could cause issues if it is set to
+                                          only authenticate when these are present. This is only used for NTLMv2 authentication.
+
+        Message Attributes (Attributes not used to compute the message structure):
+            signature: An 8-byte character array that MUST contain the ASCII string 'NTLMSSP\0'
+            message_type: A 32-bit unsigned integer that indicates the message type. This field must be set to 0x00000003
+            negotiate_flags: A NEGOTIATE strucutre that contains a set of bit flags. These flags are the choices the client has made from the CHALLENGE_MESSAGE options
+            version: Contains the windows version info of the client. It is used only debugging purposes and are only set when NTLMSSP_NEGOTIATE_VERSION flag is set
+            mic: The message integrity for the NEGOTIATE_MESSAGE, CHALLENGE_MESSAGE and AUTHENTICATE_MESSAGE
+            lm_challenge_response: An LM_RESPONSE of LMv2_RESPONSE structure that contains the computed LM response to the challenge
+            nt_challenge_response: An NTLM_RESPONSE or NTLMv2_RESPONSE structure that contains the computed NT response to the challenge
+            domain_name: The domain or computer name hosting the user account, MUST be encoded in the negotiated character set
+            user_name: The name of the user to be authenticated, MUST be encoded in the negotiated character set
+            workstation: The name of the computer to which the user is logged on, MUST Be encoded in the negotiated character set
+            encrypted_random_session_key: The client's encrypted random session key
+
+        Non-Message Attributes (Attributes not used to compute the message structure):
+            exported_session_key: A randomly generated session key based on other keys, used to derive the SIGNKEY and SEALKEY
+            target_info: The AV_PAIR structure used in the nt response calculation
+    """
+    def __init__(self, user_name, password, domain_name, workstation, challenge_message, ntlm_compatibility, server_certificate_hash):
+        self.signature = NTLM_SIGNATURE
+        self.message_type = struct.pack('<L', MessageTypes.NTLM_AUTHENTICATE)
+        self.negotiate_flags = challenge_message.negotiate_flags
+        self.version = get_version(self.negotiate_flags)
+        self.mic = None
+
+        if domain_name is None:
+            self.domain_name = ''
+        else:
+            self.domain_name = domain_name
+
+        if workstation is None:
+            self.workstation = ''
+        else:
+            self.workstation = workstation
+
+        if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_UNICODE:
+            self.negotiate_flags -= NegotiateFlags.NTLMSSP_NEGOTIATE_OEM
+            encoding_value = 'utf-16-le'
+        else:
+            encoding_value = 'ascii'
+
+        self.domain_name = self.domain_name.encode(encoding_value)
+        self.user_name = user_name.encode(encoding_value)
+        self.workstation = self.workstation.encode(encoding_value)
+
+        compute_response = ComputeResponse(user_name, password, domain_name, challenge_message,
+                                           ntlm_compatibility)
+
+        self.lm_challenge_response = compute_response.get_lm_challenge_response()
+        self.nt_challenge_response, key_exchange_key, target_info = compute_response.get_nt_challenge_response(
+            self.lm_challenge_response, server_certificate_hash)
+        self.target_info = target_info
+
+        if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_KEY_EXCH:
+            self.exported_session_key = get_random_export_session_key()
+
+            rc4_handle = ARC4(key_exchange_key)
+            self.encrypted_random_session_key = rc4_handle.update(self.exported_session_key)
+        else:
+            self.exported_session_key = key_exchange_key
+            self.encrypted_random_session_key = b''
+
+        self.negotiate_flags = struct.pack('<I', self.negotiate_flags)
+
+    def get_data(self):
+        if self.mic is None:
+            mic = b''
+            expected_body_length = self.EXPECTED_BODY_LENGTH
+        else:
+            mic = self.mic
+            expected_body_length = self.EXPECTED_BODY_LENGTH_WITH_MIC
+
+        payload_offset = expected_body_length
+
+        # DomainNameFields - 8 bytes
+        domain_name_len = struct.pack('<H', len(self.domain_name))
+        domain_name_max_len = struct.pack('<H', len(self.domain_name))
+        domain_name_buffer_offset = struct.pack('<I', payload_offset)
+        payload_offset += len(self.domain_name)
+
+        # UserNameFields - 8 bytes
+        user_name_len = struct.pack('<H', len(self.user_name))
+        user_name_max_len = struct.pack('<H', len(self.user_name))
+        user_name_buffer_offset = struct.pack('<I', payload_offset)
+        payload_offset += len(self.user_name)
+
+        # WorkstatonFields - 8 bytes
+        workstation_len = struct.pack('<H', len(self.workstation))
+        workstation_max_len = struct.pack('<H', len(self.workstation))
+        workstation_buffer_offset = struct.pack('<I', payload_offset)
+        payload_offset += len(self.workstation)
+
+        # LmChallengeResponseFields - 8 bytes
+        lm_challenge_response_len = struct.pack('<H', len(self.lm_challenge_response))
+        lm_challenge_response_max_len = struct.pack('<H', len(self.lm_challenge_response))
+        lm_challenge_response_buffer_offset = struct.pack('<I', payload_offset)
+        payload_offset += len(self.lm_challenge_response)
+
+        # NtChallengeResponseFields - 8 bytes
+        nt_challenge_response_len = struct.pack('<H', len(self.nt_challenge_response))
+        nt_challenge_response_max_len = struct.pack('<H', len(self.nt_challenge_response))
+        nt_challenge_response_buffer_offset = struct.pack('<I', payload_offset)
+        payload_offset += len(self.nt_challenge_response)
+
+        # EncryptedRandomSessionKeyFields - 8 bytes
+        encrypted_random_session_key_len = struct.pack('<H', len(self.encrypted_random_session_key))
+        encrypted_random_session_key_max_len = struct.pack('<H', len(self.encrypted_random_session_key))
+        encrypted_random_session_key_buffer_offset = struct.pack('<I', payload_offset)
+        payload_offset += len(self.encrypted_random_session_key)
+
+        # Payload - variable length
+        payload = self.domain_name
+        payload += self.user_name
+        payload += self.workstation
+        payload += self.lm_challenge_response
+        payload += self.nt_challenge_response
+        payload += self.encrypted_random_session_key
+
+        msg3 = self.signature
+        msg3 += self.message_type
+        msg3 += lm_challenge_response_len + lm_challenge_response_max_len + lm_challenge_response_buffer_offset
+        msg3 += nt_challenge_response_len + nt_challenge_response_max_len + nt_challenge_response_buffer_offset
+        msg3 += domain_name_len + domain_name_max_len + domain_name_buffer_offset
+        msg3 += user_name_len + user_name_max_len + user_name_buffer_offset
+        msg3 += workstation_len + workstation_max_len + workstation_buffer_offset
+        msg3 += encrypted_random_session_key_len + encrypted_random_session_key_max_len + encrypted_random_session_key_buffer_offset
+        msg3 += self.negotiate_flags
+        msg3 += self.version
+        msg3 += mic
+
+        # Adding the payload data to the message
+        msg3 += payload
+
+        return msg3
+
+    def add_mic(self, negotiate_message, challenge_message):
+        if self.target_info is not None:
+            av_flags = self.target_info[TargetInfo.MSV_AV_FLAGS]
+
+            if av_flags is not None and av_flags[1] == struct.pack("<L", AvFlags.MIC_PROVIDED):
+                self.mic = struct.pack("<IIII", 0, 0, 0, 0)
+                negotiate_data = negotiate_message.get_data()
+                challenge_data = challenge_message.get_data()
+                authenticate_data = self.get_data()
+
+                mic = hmac.new(self.exported_session_key,
+                               (negotiate_data + challenge_data + authenticate_data)).digest()
+                self.mic = mic
+
+def get_version(negotiate_flags):
+    # Check the negotiate_flag version is set, if it is make sure the version info is added to the data
+    if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_VERSION:
+        # TODO: Get the major and minor version of Windows instead of using default values
+        product_major_version = struct.pack('<B', 6)
+        product_minor_version = struct.pack('<B', 1)
+        product_build = struct.pack('<H', 7601)
+        version_reserved = b'\0' * 3
+        ntlm_revision_current = struct.pack('<B', 15)
+        version = product_major_version + product_minor_version + product_build + version_reserved + ntlm_revision_current
+    else:
+        version = b'\0' * 8
+
+    return version
+
+def get_random_export_session_key():
+    return os.urandom(16)

packages/wakatime/packages/ntlm_auth/ntlm.py

@@ -0,0 +1,146 @@
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+import base64
+import socket
+import struct
+from ntlm_auth.constants import NegotiateFlags
+from ntlm_auth.messages import NegotiateMessage, ChallengeMessage, AuthenticateMessage
+from ntlm_auth.session_security import SessionSecurity
+
+
+"""
+utility functions for Microsoft NTLM authentication
+
+References:
+[MS-NLMP]: NT LAN Manager (NTLM) Authentication Protocol Specification
+http://download.microsoft.com/download/a/e/6/ae6e4142-aa58-45c6-8dcf-a657e5900cd3/%5BMS-NLMP%5D.pdf
+
+[MS-NTHT]: NTLM Over HTTP Protocol Specification
+http://download.microsoft.com/download/a/e/6/ae6e4142-aa58-45c6-8dcf-a657e5900cd3/%5BMS-NTHT%5D.pdf
+
+Cntlm Authentication Proxy
+http://cntlm.awk.cz/
+
+NTLM Authorization Proxy Server
+http://sourceforge.net/projects/ntlmaps/
+
+Optimized Attack for NTLM2 Session Response
+http://www.blackhat.com/presentations/bh-asia-04/bh-jp-04-pdfs/bh-jp-04-seki.pdf
+"""
+
+class Ntlm(object):
+    """
+    Initialises the NTLM context to use when sending and receiving messages to and from the server. You should be
+    using this object as it supports NTLMv2 authenticate and it easier to use than before. It also brings in the
+    ability to use signing and sealing with session_security and generate a MIC structure.
+
+    :param ntlm_compatibility: The Lan Manager Compatibility Level to use withe the auth message - Default 3
+                                This is set by an Administrator in the registry key
+                                'HKLM\SYSTEM\CurrentControlSet\Control\Lsa\LmCompatibilityLevel'
+                                The values correspond to the following;
+                                    0 : LM and NTLMv1
+                                    1 : LM, NTLMv1 and NTLMv1 with Extended Session Security
+                                    2 : NTLMv1 and NTLMv1 with Extended Session Security
+                                    3-5 : NTLMv2 Only
+                                Note: Values 3 to 5 are no different as the client supports the same types
+
+    Attributes:
+        negotiate_flags: A NEGOTIATE structure that contains a set of bit flags. These flags are the options the client supports and are sent in the negotiate_message
+        ntlm_compatibility: The Lan Manager Compatibility Level, same as the input if supplied
+        negotiate_message: A NegotiateMessage object that is sent to the server
+        challenge_message: A ChallengeMessage object that has been created from the server response
+        authenticate_message: An AuthenticateMessage object that is sent to the server based on the ChallengeMessage
+        session_security: A SessionSecurity structure that can be used to sign and seal messages sent after the authentication challenge
+    """
+    def __init__(self, ntlm_compatibility=3):
+        self.ntlm_compatibility = ntlm_compatibility
+
+        # Setting up our flags so the challenge message returns the target info block if supported
+        self.negotiate_flags = NegotiateFlags.NTLMSSP_NEGOTIATE_TARGET_INFO | \
+                               NegotiateFlags.NTLMSSP_NEGOTIATE_128 | \
+                               NegotiateFlags.NTLMSSP_NEGOTIATE_56 | \
+                               NegotiateFlags.NTLMSSP_NEGOTIATE_UNICODE | \
+                               NegotiateFlags.NTLMSSP_NEGOTIATE_VERSION | \
+                               NegotiateFlags.NTLMSSP_NEGOTIATE_KEY_EXCH | \
+                               NegotiateFlags.NTLMSSP_NEGOTIATE_ALWAYS_SIGN | \
+                               NegotiateFlags.NTLMSSP_NEGOTIATE_SIGN | \
+                               NegotiateFlags.NTLMSSP_NEGOTIATE_SEAL
+
+        # Setting the message types based on the ntlm_compatibility level
+        self._set_ntlm_compatibility_flags(self.ntlm_compatibility)
+
+        self.negotiate_message = None
+        self.challenge_message = None
+        self.authenticate_message = None
+        self.session_security = None
+
+
+    def create_negotiate_message(self, domain_name=None, workstation=None):
+        """
+        Create an NTLM NEGOTIATE_MESSAGE
+
+        :param domain_name: The domain name of the user account we are authenticating with, default is None
+        :param worksation: The workstation we are using to authenticate with, default is None
+        :return: A base64 encoded string of the NEGOTIATE_MESSAGE
+        """
+        self.negotiate_message = NegotiateMessage(self.negotiate_flags, domain_name, workstation)
+
+        return base64.b64encode(self.negotiate_message.get_data())
+
+    def parse_challenge_message(self, msg2):
+        """
+        Parse the NTLM CHALLENGE_MESSAGE from the server and add it to the Ntlm context fields
+
+        :param msg2: A base64 encoded string of the CHALLENGE_MESSAGE
+        """
+        msg2 = base64.b64decode(msg2)
+        self.challenge_message = ChallengeMessage(msg2)
+
+    def create_authenticate_message(self, user_name, password, domain_name=None, workstation=None, server_certificate_hash=None):
+        """
+        Create an NTLM AUTHENTICATE_MESSAGE based on the Ntlm context and the previous messages sent and received
+
+        :param user_name: The user name of the user we are trying to authenticate with
+        :param password: The password of the user we are trying to authenticate with
+        :param domain_name: The domain name of the user account we are authenticated with, default is None
+        :param workstation: The workstation we are using to authenticate with, default is None
+        :param server_certificate_hash: The SHA256 hash string of the server certificate (DER encoded) NTLM is authenticating to. Used for Channel
+                                        Binding Tokens. If nothing is supplied then the CBT hash will not be sent. See messages.py AuthenticateMessage
+                                        for more details
+        :return: A base64 encoded string of the AUTHENTICATE_MESSAGE
+        """
+        self.authenticate_message = AuthenticateMessage(user_name, password, domain_name, workstation,
+                                                        self.challenge_message, self.ntlm_compatibility,
+                                                        server_certificate_hash)
+        self.authenticate_message.add_mic(self.negotiate_message, self.challenge_message)
+
+        # Setups up the session_security context used to sign and seal messages if wanted
+        if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_SEAL or self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_SIGN:
+            self.session_security = SessionSecurity(struct.unpack("<I", self.authenticate_message.negotiate_flags)[0],
+                                                    self.authenticate_message.exported_session_key)
+
+        return base64.b64encode(self.authenticate_message.get_data())
+
+    def _set_ntlm_compatibility_flags(self, ntlm_compatibility):
+        if (ntlm_compatibility >= 0) and (ntlm_compatibility <= 5):
+            if ntlm_compatibility == 0:
+                self.negotiate_flags |= NegotiateFlags.NTLMSSP_NEGOTIATE_NTLM | \
+                                        NegotiateFlags.NTLMSSP_NEGOTIATE_LM_KEY
+            elif ntlm_compatibility == 1:
+                self.negotiate_flags |= NegotiateFlags.NTLMSSP_NEGOTIATE_NTLM | \
+                                        NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY
+            else:
+                self.negotiate_flags |= NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY
+        else:
+            raise Exception("Unknown ntlm_compatibility level - expecting value between 0 and 5")

packages/wakatime/packages/ntlm_auth/rc4.py

@@ -0,0 +1,51 @@
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+class ARC4(object):
+    state = None
+    i = 0
+    j = 0
+
+    def __init__(self, key):
+        # Split up the key into a list
+        if isinstance(key, str):
+            key = [ord(c) for c in key]
+        else:
+            key = [c for c in key]
+
+        #Key-scheduling algorithm (KSA)
+        self.state = [n for n in range(256)]
+        j = 0
+        for i in range(256):
+            j = (j + self.state[i] + key[i % len(key)]) % 256
+            self.state[i], self.state[j] = self.state[j], self.state[i]
+
+    def update(self, value):
+        chars = []
+        random_gen = self._random_generator()
+        for char in value:
+            if isinstance(value, str):
+                byte = ord(char)
+            else:
+                byte = char
+            updated_byte = byte ^ next(random_gen)
+            chars.append(updated_byte)
+        return bytes(bytearray(chars))
+
+    def _random_generator(self):
+        #Pseudo-Random Generation Algorithm (PRGA)
+        while True:
+            self.i = (self.i + 1) % 256
+            self.j = (self.j + self.state[self.i]) % 256
+            self.state[self.i], self.state[self.j] = self.state[self.j], self.state[self.i]
+            yield self.state[(self.state[self.i] + self.state[self.j]) % 256]

packages/wakatime/packages/ntlm_auth/session_security.py

@@ -0,0 +1,250 @@
+# This library is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation, either
+# version 3 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library.  If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
+
+import binascii
+import hmac
+import struct
+import ntlm_auth.compute_keys as compkeys
+from ntlm_auth.constants import NegotiateFlags, SignSealConstants
+from ntlm_auth.rc4 import ARC4
+
+
+class _NtlmMessageSignature1(object):
+    EXPECTED_BODY_LENGTH = 16
+
+    """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.2.9.1 NTLMSSP_MESSAGE_SIGNATURE
+        This version of the NTLMSSP_MESSAGE_SIGNATURE structure MUST be used when the
+        NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY flag is not negotiated.
+
+        :param random_pad: A 4-byte array that contains the random pad for the emssage
+        :param checksum: A 4-byte array that contains the checksum for the message
+        :param seq_num: A 32-bit unsigned integer that contains the NTLM sequence number for this application message
+    """
+    def __init__(self, random_pad, checksum, seq_num):
+        self.version = struct.pack("<I", 1)
+        self.random_pad = random_pad
+        self.checksum = checksum
+        self.seq_num = seq_num
+
+    def get_data(self):
+        signature = self.version
+        signature += self.random_pad
+        signature += self.checksum
+        signature += self.seq_num
+
+        assert self.EXPECTED_BODY_LENGTH == len(signature), "BODY_LENGTH: %d != signature: %d" % (
+        self.EXPECTED_BODY_LENGTH, len(signature))
+
+        return signature
+
+class _NtlmMessageSignature2(object):
+    EXPECTED_BODY_LENGTH = 16
+
+    """
+        [MS-NLMP] v28.0 2016-07-14
+
+        2.2.2.9.2 NTLMSSP_MESSAGE_SIGNATURE for Extended Session Security
+        This version of the NTLMSSP_MESSAGE_SIGNATURE structure MUST be used when the
+        NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY flag is negotiated
+
+        :param checksum: An 8-byte array that contains the checksum for the message
+        :param seq_num: A 32-bit unsigned integer that contains the NTLM sequence number for this application message
+    """
+
+    def __init__(self, checksum, seq_num):
+        self.version = struct.pack("<I", 1)
+        self.checksum = checksum
+        self.seq_num = seq_num
+
+    def get_data(self):
+        signature = self.version
+        signature += self.checksum
+        signature += self.seq_num
+
+        assert self.EXPECTED_BODY_LENGTH == len(signature), "BODY_LENGTH: %d != signature: %d" % (
+            self.EXPECTED_BODY_LENGTH, len(signature))
+
+        return signature
+
+class SessionSecurity(object):
+    """
+    Initialises a security session context that can be used by libraries that call ntlm-auth to sign and seal
+    messages send to the server as well as verify and unseal messages that have been received from the server.
+    This is similar to the GSS_Wrap functions specified in the MS-NLMP document which does the same task.
+
+    :param negotiate_flags: The negotiate flag structure that has been negotiated with the server
+    :param exported_session_key: A 128-bit session key used to derive signing and sealing keys
+    :param source: The source of the message, only used in test scenarios when testing out a server sealing and unsealing
+    """
+    def __init__(self, negotiate_flags, exported_session_key, source="client"):
+        self.negotiate_flags = negotiate_flags
+        self.outgoing_seq_num = 0
+        self.incoming_seq_num = 0
+
+        client_sealing_key = compkeys.get_seal_key(self.negotiate_flags, exported_session_key, SignSealConstants.CLIENT_SEALING)
+        server_sealing_key = compkeys.get_seal_key(self.negotiate_flags, exported_session_key, SignSealConstants.SERVER_SEALING)
+
+        if source == "client":
+            self.outgoing_signing_key = compkeys.get_sign_key(exported_session_key, SignSealConstants.CLIENT_SIGNING)
+            self.incoming_signing_key = compkeys.get_sign_key(exported_session_key, SignSealConstants.SERVER_SIGNING)
+            self.outgoing_handle = ARC4(client_sealing_key)
+            self.incoming_handle = ARC4(server_sealing_key)
+        elif source == "server":
+            self.outgoing_signing_key = compkeys.get_sign_key(exported_session_key, SignSealConstants.SERVER_SIGNING)
+            self.incoming_signing_key = compkeys.get_sign_key(exported_session_key, SignSealConstants.CLIENT_SIGNING)
+            self.outgoing_handle = ARC4(server_sealing_key)
+            self.incoming_handle = ARC4(client_sealing_key)
+        else:
+            raise Exception("Invalid source parameter %s, must be client or server" % source)
+
+    def wrap(self, message):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        3.4.6 GSS_WrapEx()
+        Emulates the GSS_Wrap() implementation to sign and seal messages if the correct flags
+        are set.
+
+        @param message: The message data that will be wrapped
+        @return message: The message that has been sealed if flags are set
+        @return signature: The signature of the message, None if flags are not set
+        """
+        if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_SEAL:
+            encrypted_message = self._seal_message(message)
+            signature = self._get_signature(message)
+            message = encrypted_message
+
+        elif self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_SIGN:
+            signature = self._get_signature(message)
+        else:
+            signature = None
+
+        return message, signature
+
+    def unwrap(self, message, signature):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        3.4.7 GSS_UnwrapEx()
+        Emulates the GSS_Unwrap() implementation to unseal messages and verify the signature
+        sent matches what has been computed locally. Will throw an Exception if the signature
+        doesn't match
+
+        @param message: The message data received from the server
+        @param signature: The signature of the message
+        @return message: The message that has been unsealed if flags are set
+        """
+        if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_SEAL:
+            message = self._unseal_message(message)
+            self._verify_signature(message, signature)
+
+        elif self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_SIGN:
+            self._verify_signature(message, signature)
+
+        return message
+
+    def _seal_message(self, message):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        3.4.3 Message Confidentiality
+        Will generate an encrypted message using RC4 based on the ClientSealingKey
+
+        @param message: The message to be sealed (encrypted)
+        @return encrypted_message: The encrypted message
+        """
+        encrypted_message = self.outgoing_handle.update(message)
+        return encrypted_message
+
+    def _unseal_message(self, message):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        3.4.3 Message Confidentiality
+        Will generate a dencrypted message using RC4 based on the ServerSealingKey
+
+        @param message: The message to be unsealed (dencrypted)
+        @return decrypted_message: The decrypted message
+        """
+        decrypted_message = self.incoming_handle.update(message)
+        return decrypted_message
+
+    def _get_signature(self, message):
+        """
+        [MS-NLMP] v28.0 2016-07-14
+
+        3.4.4 Message Signature Functions
+        Will create the signature based on the message to send to the server. Depending on the negotiate_flags
+        set this could either be an NTLMv1 signature or NTLMv2 with Extended Session Security signature.
+
+        @param message: The message data that will be signed
+        @return signature: Either _NtlmMessageSignature1 or _NtlmMessageSignature2 depending on the flags set
+        """
+        signature = calc_signature(message, self.negotiate_flags, self.outgoing_signing_key, self.outgoing_seq_num, self.outgoing_handle)
+        self.outgoing_seq_num += 1
+
+        return signature.get_data()
+
+    def _verify_signature(self, message, signature):
+        """
+        Will verify that the signature received from the server matches up with the expected signature
+        computed locally. Will throw an exception if they do not match
+
+        @param message: The message data that is received from the server
+        @param signature: The signature of the message received from the server
+        """
+        if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY:
+            actual_checksum = signature[4:12]
+            actual_seq_num = struct.unpack("<I", signature[12:16])[0]
+        else:
+            actual_checksum = signature[8:12]
+            actual_seq_num = struct.unpack("<I", signature[12:16])[0]
+
+        expected_signature = calc_signature(message, self.negotiate_flags, self.incoming_signing_key, self.incoming_seq_num, self.incoming_handle)
+        expected_checksum = expected_signature.checksum
+        expected_seq_num = struct.unpack("<I", expected_signature.seq_num)[0]
+
+        if actual_checksum != expected_checksum:
+            raise Exception("The signature checksum does not match, message has been altered")
+
+        if actual_seq_num != expected_seq_num:
+            raise Exception("The signature sequence number does not match up, message not received in the correct sequence")
+
+        self.incoming_seq_num += 1
+
+
+def calc_signature(message, negotiate_flags, signing_key, seq_num, handle):
+    seq_num = struct.pack("<I", seq_num)
+    if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY:
+        checksum_hmac = hmac.new(signing_key, seq_num + message)
+        if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_KEY_EXCH:
+            checksum = handle.update(checksum_hmac.digest()[:8])
+        else:
+            checksum = checksum_hmac.digest()[:8]
+
+        signature = _NtlmMessageSignature2(checksum, seq_num)
+
+    else:
+        message_crc = binascii.crc32(message) % (1 << 32)
+        checksum = struct.pack("<I", message_crc)
+        random_pad = handle.update(struct.pack("<I", 0))
+        checksum = handle.update(checksum)
+        seq_num = handle.update(seq_num)
+        random_pad = struct.pack("<I", 0)
+
+        signature = _NtlmMessageSignature1(random_pad, checksum, seq_num)
+
+    return signature

packages/wakatime/packages/ntlm_auth/target_info.py

@@ -0,0 +1,68 @@
+"""
+    Original Author: Ian Clegg
+    Project: ntlmlib
+    URL: https://github.com/ianclegg/ntlmlib
+    License: Apache 2.0 License
+    Notes: Most of this code has been copied from the messages.py in the ntlmlib repo.
+    Some minor changes such as the name of the AV Pairs and extra comments have been added.
+"""
+
+import struct
+try:
+    from collections import OrderedDict
+except ImportError:
+    from ordereddict import OrderedDict
+
+class TargetInfo(object):
+    MSV_AV_EOL                  = 0x00
+    MSV_AV_NB_COMPUTER_NAME     = 0x01
+    MSV_AV_NB_DOMAIN_NAME       = 0x02
+    MSV_AV_DNS_COMPUTER_NAME    = 0x03
+    MSV_AV_DNS_DOMAIN_NAME      = 0x04
+    MSV_AV_DNS_TREE_NAME        = 0x05
+    MSV_AV_FLAGS                = 0x06
+    MSV_AV_TIMESTAMP            = 0x07
+    MSV_AV_SINGLE_HOST          = 0x08
+    MSV_AV_TARGET_NAME          = 0x09
+    MSV_AV_CHANNEL_BINDINGS     = 0x0a
+
+    def __init__(self, data=None):
+        self.fields = OrderedDict()
+        if data is not None:
+            self.from_string(data)
+
+    def __setitem__(self, key, value):
+        self.fields[key] = (len(value), value)
+
+    def __getitem__(self, key):
+        if key in self.fields:
+           return self.fields[key]
+        return None
+
+    def __delitem__(self, key):
+        del self.fields[key]
+
+    def from_string(self, data):
+        attribute_type = 0xff
+        while attribute_type is not TargetInfo.MSV_AV_EOL:
+            # Parse the Attribute Value pair from the structure
+            attribute_type = struct.unpack('<H', data[:struct.calcsize('<H')])[0]
+            data = data[struct.calcsize('<H'):]
+            length = struct.unpack('<H', data[:struct.calcsize('<H')])[0]
+            data = data[struct.calcsize('<H'):]
+            # Add a new field to the object for the parse attribute value
+            self.fields[attribute_type] = (length, data[:length])
+            data = data[length:]
+
+    def get_data(self):
+        if TargetInfo.MSV_AV_EOL in self.fields:
+            del self.fields[TargetInfo.MSV_AV_EOL]
+
+        data = b''
+        for i in self.fields.keys():
+            data += struct.pack('<HH', i, self[i][0])
+            data += self[i][1]
+
+        # end with a NTLMSSP_AV_EOL
+        data += struct.pack('<HH', TargetInfo.MSV_AV_EOL, 0)
+        return data

packages/wakatime/packages/pygments/__init__.py

@@ -22,20 +22,18 @@
     .. _Pygments tip:
        http://bitbucket.org/birkenfeld/pygments-main/get/tip.zip#egg=Pygments-dev
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
-
-__version__ = '1.6'
-__docformat__ = 'restructuredtext'
-
-__all__ = ['lex', 'format', 'highlight']
-
-
 import sys
 
 from pygments.util import StringIO, BytesIO
 
+__version__ = '2.2.0'
+__docformat__ = 'restructuredtext'
+
+__all__ = ['lex', 'format', 'highlight']
+
 
 def lex(code, lexer):
     """
@@ -44,14 +42,15 @@ def lex(code, lexer):
     try:
         return lexer.get_tokens(code)
     except TypeError as err:
-        if isinstance(err.args[0], str) and \
-           'unbound method get_tokens' in err.args[0]:
+        if (isinstance(err.args[0], str) and
+            ('unbound method get_tokens' in err.args[0] or
+             'missing 1 required positional argument' in err.args[0])):
             raise TypeError('lex() argument must be a lexer instance, '
                             'not a class')
         raise
 
 
-def format(tokens, formatter, outfile=None):
+def format(tokens, formatter, outfile=None):  # pylint: disable=redefined-builtin
     """
     Format a tokenlist ``tokens`` with the formatter ``formatter``.
 
@@ -61,15 +60,15 @@ def format(tokens, formatter, outfile=None):
     """
     try:
         if not outfile:
-            #print formatter, 'using', formatter.encoding
-            realoutfile = formatter.encoding and BytesIO() or StringIO()
+            realoutfile = getattr(formatter, 'encoding', None) and BytesIO() or StringIO()
             formatter.format(tokens, realoutfile)
             return realoutfile.getvalue()
         else:
             formatter.format(tokens, outfile)
     except TypeError as err:
-        if isinstance(err.args[0], str) and \
-           'unbound method format' in err.args[0]:
+        if (isinstance(err.args[0], str) and
+            ('unbound method format' in err.args[0] or
+             'missing 1 required positional argument' in err.args[0])):
             raise TypeError('format() argument must be a formatter instance, '
                             'not a class')
         raise
@@ -86,6 +85,6 @@ def highlight(code, lexer, formatter, outfile=None):
     return format(lex(code, lexer), formatter, outfile)
 
 
-if __name__ == '__main__':
+if __name__ == '__main__':  # pragma: no cover
     from pygments.cmdline import main
     sys.exit(main(sys.argv))

packages/wakatime/packages/pygments/cmdline.py

@@ -5,27 +5,35 @@
 
     Command line interface.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
+
+from __future__ import print_function
+
+import os
 import sys
 import getopt
 from textwrap import dedent
 
 from pygments import __version__, highlight
-from pygments.util import ClassNotFound, OptionError, docstring_headline
-from pygments.lexers import get_all_lexers, get_lexer_by_name, get_lexer_for_filename, \
-     find_lexer_class, guess_lexer, TextLexer
+from pygments.util import ClassNotFound, OptionError, docstring_headline, \
+    guess_decode, guess_decode_from_terminal, terminal_encoding
+from pygments.lexers import get_all_lexers, get_lexer_by_name, guess_lexer, \
+    load_lexer_from_file, get_lexer_for_filename, find_lexer_class_for_filename
+from pygments.lexers.special import TextLexer
+from pygments.formatters.latex import LatexEmbeddedLexer, LatexFormatter
 from pygments.formatters import get_all_formatters, get_formatter_by_name, \
-     get_formatter_for_filename, find_formatter_class, \
-     TerminalFormatter  # pylint:disable-msg=E0611
+    load_formatter_from_file, get_formatter_for_filename, find_formatter_class
+from pygments.formatters.terminal import TerminalFormatter
+from pygments.formatters.terminal256 import Terminal256Formatter
 from pygments.filters import get_all_filters, find_filter_class
 from pygments.styles import get_all_styles, get_style_by_name
 
 
 USAGE = """\
 Usage: %s [-l <lexer> | -g] [-F <filter>[:<options>]] [-f <formatter>]
-          [-O <options>] [-P <option=value>] [-o <outfile>] [<infile>]
+          [-O <options>] [-P <option=value>] [-s] [-v] [-x] [-o <outfile>] [<infile>]
 
        %s -S <style> -f <formatter> [-a <arg>] [-O <options>] [-P <option=value>]
        %s -L [<which> ...]
@@ -37,6 +45,10 @@ Highlight the input file and write the result to <outfile>.
 
 If no input file is given, use stdin, if -o is not given, use stdout.
 
+If -s is passed, lexing will be done in "streaming" mode, reading and
+highlighting one line at a time.  This will only work properly with
+lexers that have no constructs spanning multiple lines!
+
 <lexer> is a lexer name (query all lexer names with -L). If -l is not
 given, the lexer is guessed from the extension of the input file name
 (this obviously doesn't work if the input is stdin).  If -g is passed,
@@ -47,6 +59,14 @@ Likewise, <formatter> is a formatter name, and will be guessed from
 the extension of the output file name. If no output file is given,
 the terminal formatter will be used by default.
 
+The additional option -x allows custom lexers and formatters to be
+loaded from a .py file relative to the current working directory. For
+example, ``-l ./customlexer.py -x``. By default, this option expects a
+file with a class named CustomLexer or CustomFormatter; you can also
+specify your own class name with a colon (``-l ./lexer.py:MyLexer``).
+Users should be very careful not to use this option with untrusted files,
+because it will import and run them.
+
 With the -O option, you can give the lexer and formatter a comma-
 separated list of options, e.g. ``-O bg=light,python=cool``.
 
@@ -76,6 +96,14 @@ If no specific lexer can be determined "text" is returned.
 The -H option prints detailed help for the object <name> of type <type>,
 where <type> is one of "lexer", "formatter" or "filter".
 
+The -s option processes lines one at a time until EOF, rather than
+waiting to process the entire file.  This only works for stdin, and
+is intended for streaming input such as you get from 'tail -f'.
+Example usage: "tail -f sql.log | pygmentize -s -l sql"
+
+The -v option prints a detailed traceback on unhandled exceptions,
+which is useful for debugging and bug reports.
+
 The -h option prints this help.
 The -V option prints the package version.
 """
@@ -86,13 +114,13 @@ def _parse_options(o_strs):
     if not o_strs:
         return opts
     for o_str in o_strs:
-        if not o_str:
+        if not o_str.strip():
             continue
         o_args = o_str.split(',')
         for o_arg in o_args:
             o_arg = o_arg.strip()
             try:
-                o_key, o_val = o_arg.split('=')
+                o_key, o_val = o_arg.split('=', 1)
                 o_key = o_key.strip()
                 o_val = o_val.strip()
             except ValueError:
@@ -118,7 +146,7 @@ def _parse_filters(f_strs):
 def _print_help(what, name):
     try:
         if what == 'lexer':
-            cls = find_lexer_class(name)
+            cls = get_lexer_by_name(name)
             print("Help on the %s lexer:" % cls.name)
             print(dedent(cls.__doc__))
         elif what == 'formatter':
@@ -129,8 +157,10 @@ def _print_help(what, name):
             cls = find_filter_class(name)
             print("Help on the %s filter:" % name)
             print(dedent(cls.__doc__))
-    except AttributeError:
+        return 0
+    except (AttributeError, ValueError):
         print("%s not found!" % what, file=sys.stderr)
+        return 1
 
 
 def _print_list(what):
@@ -184,27 +214,7 @@ def _print_list(what):
             print("    %s" % docstring_headline(cls))
 
 
-def main(args=sys.argv):
-    """
-    Main command line entry point.
-    """
-    # pylint: disable-msg=R0911,R0912,R0915
-
-    usage = USAGE % ((args[0],) * 6)
-
-    if sys.platform in ['win32', 'cygwin']:
-        try:
-            # Provide coloring under Windows, if possible
-            import colorama
-            colorama.init()
-        except ImportError:
-            pass
-
-    try:
-        popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:hVHg")
-    except getopt.GetoptError as err:
-        print(usage, file=sys.stderr)
-        return 2
+def main_inner(popts, args, usage):
     opts = {}
     O_opts = []
     P_opts = []
@@ -218,16 +228,12 @@ def main(args=sys.argv):
             F_opts.append(arg)
         opts[opt] = arg
 
-    if not opts and not args:
-        print(usage)
-        return 0
-
     if opts.pop('-h', None) is not None:
         print(usage)
         return 0
 
     if opts.pop('-V', None) is not None:
-        print('Pygments version %s, (c) 2006-2013 by Georg Brandl.' % __version__)
+        print('Pygments version %s, (c) 2006-2017 by Georg Brandl.' % __version__)
         return 0
 
     # handle ``pygmentize -L``
@@ -252,13 +258,12 @@ def main(args=sys.argv):
             print(usage, file=sys.stderr)
             return 2
 
-        what, name = args
+        what, name = args  # pylint: disable=unbalanced-tuple-unpacking
         if what not in ('lexer', 'formatter', 'filter'):
             print(usage, file=sys.stderr)
             return 2
 
-        _print_help(what, name)
-        return 0
+        return _print_help(what, name)
 
     # parse -O options
     parsed_opts = _parse_options(O_opts)
@@ -274,16 +279,16 @@ def main(args=sys.argv):
             parsed_opts[name] = value
     opts.pop('-P', None)
 
+    # encodings
+    inencoding = parsed_opts.get('inencoding', parsed_opts.get('encoding'))
+    outencoding = parsed_opts.get('outencoding', parsed_opts.get('encoding'))
+
     # handle ``pygmentize -N``
     infn = opts.pop('-N', None)
     if infn is not None:
-        try:
-            lexer = get_lexer_for_filename(infn, **parsed_opts)
-        except ClassNotFound as err:
-            lexer = TextLexer()
-        except OptionError as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
+        lexer = find_lexer_class_for_filename(infn)
+        if lexer is None:
+            lexer = TextLexer
 
         print(lexer.aliases[0])
         return 0
@@ -307,12 +312,7 @@ def main(args=sys.argv):
             print(err, file=sys.stderr)
             return 1
 
-        arg = a_opt or ''
-        try:
-            print(fmter.get_style_defs(arg))
-        except Exception as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
+        print(fmter.get_style_defs(a_opt or ''))
         return 0
 
     # if no -S is given, -a is not allowed
@@ -324,54 +324,60 @@ def main(args=sys.argv):
     F_opts = _parse_filters(F_opts)
     opts.pop('-F', None)
 
-    # select formatter
-    outfn = opts.pop('-o', None)
-    fmter = opts.pop('-f', None)
-    if fmter:
-        try:
-            fmter = get_formatter_by_name(fmter, **parsed_opts)
-        except (OptionError, ClassNotFound) as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
+    allow_custom_lexer_formatter = False
+    # -x: allow custom (eXternal) lexers and formatters
+    if opts.pop('-x', None) is not None:
+        allow_custom_lexer_formatter = True
 
-    if outfn:
-        if not fmter:
+    # select lexer
+    lexer = None
+
+    # given by name?
+    lexername = opts.pop('-l', None)
+    if lexername:
+        # custom lexer, located relative to user's cwd
+        if allow_custom_lexer_formatter and '.py' in lexername:
             try:
-                fmter = get_formatter_for_filename(outfn, **parsed_opts)
+                if ':' in lexername:
+                    filename, name = lexername.rsplit(':', 1)
+                    lexer = load_lexer_from_file(filename, name,
+                                                 **parsed_opts)
+                else:
+                    lexer = load_lexer_from_file(lexername, **parsed_opts)
+            except ClassNotFound as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
+        else:
+            try:
+                lexer = get_lexer_by_name(lexername, **parsed_opts)
             except (OptionError, ClassNotFound) as err:
                 print('Error:', err, file=sys.stderr)
                 return 1
-        try:
-            outfile = open(outfn, 'wb')
-        except Exception as err:
-            print('Error: cannot open outfile:', err, file=sys.stderr)
-            return 1
-    else:
-        if not fmter:
-            fmter = TerminalFormatter(**parsed_opts)
-        outfile = sys.stdout
 
-    # select lexer
-    lexer = opts.pop('-l', None)
-    if lexer:
-        try:
-            lexer = get_lexer_by_name(lexer, **parsed_opts)
-        except (OptionError, ClassNotFound) as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
+    # read input code
+    code = None
 
     if args:
         if len(args) > 1:
             print(usage, file=sys.stderr)
             return 2
 
+        if '-s' in opts:
+            print('Error: -s option not usable when input file specified',
+                  file=sys.stderr)
+            return 2
+
         infn = args[0]
         try:
-            code = open(infn, 'rb').read()
+            with open(infn, 'rb') as infp:
+                code = infp.read()
         except Exception as err:
             print('Error: cannot read infile:', err, file=sys.stderr)
             return 1
+        if not inencoding:
+            code, inencoding = guess_decode(code)
 
+        # do we have to guess the lexer?
         if not lexer:
             try:
                 lexer = get_lexer_for_filename(infn, code, **parsed_opts)
@@ -388,45 +394,171 @@ def main(args=sys.argv):
                 print('Error:', err, file=sys.stderr)
                 return 1
 
-    else:
-        if '-g' in opts:
+    elif '-s' not in opts:  # treat stdin as full file (-s support is later)
+        # read code from terminal, always in binary mode since we want to
+        # decode ourselves and be tolerant with it
+        if sys.version_info > (3,):
+            # Python 3: we have to use .buffer to get a binary stream
+            code = sys.stdin.buffer.read()
+        else:
             code = sys.stdin.read()
+        if not inencoding:
+            code, inencoding = guess_decode_from_terminal(code, sys.stdin)
+            # else the lexer will do the decoding
+        if not lexer:
             try:
                 lexer = guess_lexer(code, **parsed_opts)
             except ClassNotFound:
                 lexer = TextLexer(**parsed_opts)
-        elif not lexer:
-            print('Error: no lexer name given and reading ' + \
-                                'from stdin (try using -g or -l <lexer>)', file=sys.stderr)
-            return 2
-        else:
-            code = sys.stdin.read()
 
-    # No encoding given? Use latin1 if output file given,
-    # stdin/stdout encoding otherwise.
-    # (This is a compromise, I'm not too happy with it...)
-    if 'encoding' not in parsed_opts and 'outencoding' not in parsed_opts:
-        if outfn:
-            # encoding pass-through
-            fmter.encoding = 'latin1'
+    else:  # -s option needs a lexer with -l
+        if not lexer:
+            print('Error: when using -s a lexer has to be selected with -l',
+                  file=sys.stderr)
+            return 2
+
+    # process filters
+    for fname, fopts in F_opts:
+        try:
+            lexer.add_filter(fname, **fopts)
+        except ClassNotFound as err:
+            print('Error:', err, file=sys.stderr)
+            return 1
+
+    # select formatter
+    outfn = opts.pop('-o', None)
+    fmter = opts.pop('-f', None)
+    if fmter:
+        # custom formatter, located relative to user's cwd
+        if allow_custom_lexer_formatter and '.py' in fmter:
+            try:
+                if ':' in fmter:
+                    file, fmtername = fmter.rsplit(':', 1)
+                    fmter = load_formatter_from_file(file, fmtername,
+                                                     **parsed_opts)
+                else:
+                    fmter = load_formatter_from_file(fmter, **parsed_opts)
+            except ClassNotFound as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
         else:
-            if sys.version_info < (3,):
-                # use terminal encoding; Python 3's terminals already do that
-                lexer.encoding = getattr(sys.stdin, 'encoding',
-                                         None) or 'ascii'
-                fmter.encoding = getattr(sys.stdout, 'encoding',
-                                         None) or 'ascii'
-    elif not outfn and sys.version_info > (3,):
-        # output to terminal with encoding -> use .buffer
-        outfile = sys.stdout.buffer
+            try:
+                fmter = get_formatter_by_name(fmter, **parsed_opts)
+            except (OptionError, ClassNotFound) as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
+
+    if outfn:
+        if not fmter:
+            try:
+                fmter = get_formatter_for_filename(outfn, **parsed_opts)
+            except (OptionError, ClassNotFound) as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
+        try:
+            outfile = open(outfn, 'wb')
+        except Exception as err:
+            print('Error: cannot open outfile:', err, file=sys.stderr)
+            return 1
+    else:
+        if not fmter:
+            if '256' in os.environ.get('TERM', ''):
+                fmter = Terminal256Formatter(**parsed_opts)
+            else:
+                fmter = TerminalFormatter(**parsed_opts)
+        if sys.version_info > (3,):
+            # Python 3: we have to use .buffer to get a binary stream
+            outfile = sys.stdout.buffer
+        else:
+            outfile = sys.stdout
+
+    # determine output encoding if not explicitly selected
+    if not outencoding:
+        if outfn:
+            # output file? use lexer encoding for now (can still be None)
+            fmter.encoding = inencoding
+        else:
+            # else use terminal encoding
+            fmter.encoding = terminal_encoding(sys.stdout)
+
+    # provide coloring under Windows, if possible
+    if not outfn and sys.platform in ('win32', 'cygwin') and \
+       fmter.name in ('Terminal', 'Terminal256'):  # pragma: no cover
+        # unfortunately colorama doesn't support binary streams on Py3
+        if sys.version_info > (3,):
+            from pygments.util import UnclosingTextIOWrapper
+            outfile = UnclosingTextIOWrapper(outfile, encoding=fmter.encoding)
+            fmter.encoding = None
+        try:
+            import colorama.initialise
+        except ImportError:
+            pass
+        else:
+            outfile = colorama.initialise.wrap_stream(
+                outfile, convert=None, strip=None, autoreset=False, wrap=True)
+
+    # When using the LaTeX formatter and the option `escapeinside` is
+    # specified, we need a special lexer which collects escaped text
+    # before running the chosen language lexer.
+    escapeinside = parsed_opts.get('escapeinside', '')
+    if len(escapeinside) == 2 and isinstance(fmter, LatexFormatter):
+        left = escapeinside[0]
+        right = escapeinside[1]
+        lexer = LatexEmbeddedLexer(left, right, lexer)
 
     # ... and do it!
-    try:
-        # process filters
-        for fname, fopts in F_opts:
-            lexer.add_filter(fname, **fopts)
+    if '-s' not in opts:
+        # process whole input as per normal...
         highlight(code, lexer, fmter, outfile)
-    except Exception as err:
+        return 0
+    else:
+        # line by line processing of stdin (eg: for 'tail -f')...
+        try:
+            while 1:
+                if sys.version_info > (3,):
+                    # Python 3: we have to use .buffer to get a binary stream
+                    line = sys.stdin.buffer.readline()
+                else:
+                    line = sys.stdin.readline()
+                if not line:
+                    break
+                if not inencoding:
+                    line = guess_decode_from_terminal(line, sys.stdin)[0]
+                highlight(line, lexer, fmter, outfile)
+                if hasattr(outfile, 'flush'):
+                    outfile.flush()
+            return 0
+        except KeyboardInterrupt:  # pragma: no cover
+            return 0
+
+
+def main(args=sys.argv):
+    """
+    Main command line entry point.
+    """
+    usage = USAGE % ((args[0],) * 6)
+
+    try:
+        popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:vhVHgsx")
+    except getopt.GetoptError:
+        print(usage, file=sys.stderr)
+        return 2
+
+    try:
+        return main_inner(popts, args, usage)
+    except Exception:
+        if '-v' in dict(popts):
+            print(file=sys.stderr)
+            print('*' * 65, file=sys.stderr)
+            print('An unhandled exception occurred while highlighting.',
+                  file=sys.stderr)
+            print('Please report the whole traceback to the issue tracker at',
+                  file=sys.stderr)
+            print('<https://bitbucket.org/birkenfeld/pygments-main/issues>.',
+                  file=sys.stderr)
+            print('*' * 65, file=sys.stderr)
+            print(file=sys.stderr)
+            raise
         import traceback
         info = traceback.format_exception(*sys.exc_info())
         msg = info[-1].strip()
@@ -436,6 +568,6 @@ def main(args=sys.argv):
         print(file=sys.stderr)
         print('*** Error while highlighting:', file=sys.stderr)
         print(msg, file=sys.stderr)
+        print('*** If this is a bug you want to report, please rerun with -v.',
+              file=sys.stderr)
         return 1
-
-    return 0

packages/wakatime/packages/pygments/console.py

@@ -5,25 +5,25 @@
 
     Format colored console output.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 esc = "\x1b["
 
 codes = {}
-codes[""]          = ""
-codes["reset"]     = esc + "39;49;00m"
+codes[""] = ""
+codes["reset"] = esc + "39;49;00m"
 
-codes["bold"]      = esc + "01m"
-codes["faint"]     = esc + "02m"
-codes["standout"]  = esc + "03m"
+codes["bold"] = esc + "01m"
+codes["faint"] = esc + "02m"
+codes["standout"] = esc + "03m"
 codes["underline"] = esc + "04m"
-codes["blink"]     = esc + "05m"
-codes["overline"]  = esc + "06m"
+codes["blink"] = esc + "05m"
+codes["overline"] = esc + "06m"
 
-dark_colors  = ["black", "darkred", "darkgreen", "brown", "darkblue",
-                "purple", "teal", "lightgray"]
+dark_colors = ["black", "darkred", "darkgreen", "brown", "darkblue",
+               "purple", "teal", "lightgray"]
 light_colors = ["darkgray", "red", "green", "yellow", "blue",
                 "fuchsia", "turquoise", "white"]
 
@@ -35,10 +35,10 @@ for d, l in zip(dark_colors, light_colors):
 
 del d, l, x
 
-codes["darkteal"]   = codes["turquoise"]
+codes["darkteal"] = codes["turquoise"]
 codes["darkyellow"] = codes["brown"]
-codes["fuscia"]     = codes["fuchsia"]
-codes["white"]      = codes["bold"]
+codes["fuscia"] = codes["fuchsia"]
+codes["white"] = codes["bold"]
 
 
 def reset_color():

packages/wakatime/packages/pygments/filter.py

@@ -5,7 +5,7 @@
 
     Module that implements the default filter.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -29,15 +29,15 @@ def simplefilter(f):
     Decorator that converts a function into a filter::
 
         @simplefilter
-        def lowercase(lexer, stream, options):
+        def lowercase(self, lexer, stream, options):
             for ttype, value in stream:
                 yield ttype, value.lower()
     """
     return type(f.__name__, (FunctionFilter,), {
-                'function':     f,
-                '__module__':   getattr(f, '__module__'),
-                '__doc__':      f.__doc__
-            })
+        '__module__': getattr(f, '__module__'),
+        '__doc__': f.__doc__,
+        'function': f,
+    })
 
 
 class Filter(object):
@@ -69,6 +69,6 @@ class FunctionFilter(Filter):
         Filter.__init__(self, **options)
 
     def filter(self, lexer, stream):
-        # pylint: disable-msg=E1102
+        # pylint: disable=not-callable
         for ttype, value in self.function(lexer, stream, self.options):
             yield ttype, value

packages/wakatime/packages/pygments/filters/__init__.py

@@ -6,7 +6,7 @@
     Module containing filter lookup functions and default
     filters.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -16,14 +16,12 @@ from pygments.token import String, Comment, Keyword, Name, Error, Whitespace, \
     string_to_tokentype
 from pygments.filter import Filter
 from pygments.util import get_list_opt, get_int_opt, get_bool_opt, \
-     get_choice_opt, ClassNotFound, OptionError
+     get_choice_opt, ClassNotFound, OptionError, text_type, string_types
 from pygments.plugin import find_plugin_filters
 
 
 def find_filter_class(filtername):
-    """
-    Lookup a filter by name. Return None if not found.
-    """
+    """Lookup a filter by name. Return None if not found."""
     if filtername in FILTERS:
         return FILTERS[filtername]
     for name, cls in find_plugin_filters():
@@ -33,9 +31,10 @@ def find_filter_class(filtername):
 
 
 def get_filter_by_name(filtername, **options):
-    """
-    Return an instantiated filter. Options are passed to the filter
-    initializer if wanted. Raise a ClassNotFound if not found.
+    """Return an instantiated filter.
+
+    Options are passed to the filter initializer if wanted.
+    Raise a ClassNotFound if not found.
     """
     cls = find_filter_class(filtername)
     if cls:
@@ -45,9 +44,7 @@ def get_filter_by_name(filtername, **options):
 
 
 def get_all_filters():
-    """
-    Return a generator of all filter names.
-    """
+    """Return a generator of all filter names."""
     for name in FILTERS:
         yield name
     for name, _ in find_plugin_filters():
@@ -68,8 +65,7 @@ def _replace_special(ttype, value, regex, specialttype,
 
 
 class CodeTagFilter(Filter):
-    """
-    Highlight special code tags in comments and docstrings.
+    """Highlight special code tags in comments and docstrings.
 
     Options accepted:
 
@@ -100,8 +96,7 @@ class CodeTagFilter(Filter):
 
 
 class KeywordCaseFilter(Filter):
-    """
-    Convert keywords to lowercase or uppercase or capitalize them, which
+    """Convert keywords to lowercase or uppercase or capitalize them, which
     means first letter uppercase, rest lowercase.
 
     This can be useful e.g. if you highlight Pascal code and want to adapt the
@@ -116,8 +111,9 @@ class KeywordCaseFilter(Filter):
 
     def __init__(self, **options):
         Filter.__init__(self, **options)
-        case = get_choice_opt(options, 'case', ['lower', 'upper', 'capitalize'], 'lower')
-        self.convert = getattr(str, case)
+        case = get_choice_opt(options, 'case',
+                              ['lower', 'upper', 'capitalize'], 'lower')
+        self.convert = getattr(text_type, case)
 
     def filter(self, lexer, stream):
         for ttype, value in stream:
@@ -128,8 +124,7 @@ class KeywordCaseFilter(Filter):
 
 
 class NameHighlightFilter(Filter):
-    """
-    Highlight a normal Name (and Name.*) token with a different token type.
+    """Highlight a normal Name (and Name.*) token with a different token type.
 
     Example::
 
@@ -172,9 +167,9 @@ class NameHighlightFilter(Filter):
 class ErrorToken(Exception):
     pass
 
+
 class RaiseOnErrorTokenFilter(Filter):
-    """
-    Raise an exception when the lexer generates an error token.
+    """Raise an exception when the lexer generates an error token.
 
     Options accepted:
 
@@ -182,7 +177,7 @@ class RaiseOnErrorTokenFilter(Filter):
       The exception class to raise.
       The default is `pygments.filters.ErrorToken`.
 
-    *New in Pygments 0.8.*
+    .. versionadded:: 0.8
     """
 
     def __init__(self, **options):
@@ -203,8 +198,7 @@ class RaiseOnErrorTokenFilter(Filter):
 
 
 class VisibleWhitespaceFilter(Filter):
-    """
-    Convert tabs, newlines and/or spaces to visible characters.
+    """Convert tabs, newlines and/or spaces to visible characters.
 
     Options accepted:
 
@@ -230,29 +224,31 @@ class VisibleWhitespaceFilter(Filter):
       styling the visible whitespace differently (e.g. greyed out), but it can
       disrupt background colors.  The default is ``True``.
 
-    *New in Pygments 0.8.*
+    .. versionadded:: 0.8
     """
 
     def __init__(self, **options):
         Filter.__init__(self, **options)
-        for name, default in list({'spaces': '·', 'tabs': '»', 'newlines': '¶'}.items()):
+        for name, default in [('spaces',   u'·'),
+                              ('tabs',     u'»'),
+                              ('newlines', u'¶')]:
             opt = options.get(name, False)
-            if isinstance(opt, str) and len(opt) == 1:
+            if isinstance(opt, string_types) and len(opt) == 1:
                 setattr(self, name, opt)
             else:
                 setattr(self, name, (opt and default or ''))
         tabsize = get_int_opt(options, 'tabsize', 8)
         if self.tabs:
-            self.tabs += ' '*(tabsize-1)
+            self.tabs += ' ' * (tabsize - 1)
         if self.newlines:
             self.newlines += '\n'
         self.wstt = get_bool_opt(options, 'wstokentype', True)
 
     def filter(self, lexer, stream):
         if self.wstt:
-            spaces = self.spaces or ' '
-            tabs = self.tabs or '\t'
-            newlines = self.newlines or '\n'
+            spaces = self.spaces or u' '
+            tabs = self.tabs or u'\t'
+            newlines = self.newlines or u'\n'
             regex = re.compile(r'\s')
             def replacefunc(wschar):
                 if wschar == ' ':
@@ -281,8 +277,7 @@ class VisibleWhitespaceFilter(Filter):
 
 
 class GobbleFilter(Filter):
-    """
-    Gobbles source code lines (eats initial characters).
+    """Gobbles source code lines (eats initial characters).
 
     This filter drops the first ``n`` characters off every line of code.  This
     may be useful when the source code fed to the lexer is indented by a fixed
@@ -293,7 +288,7 @@ class GobbleFilter(Filter):
     `n` : int
        The number of characters to gobble.
 
-    *New in Pygments 1.2.*
+    .. versionadded:: 1.2
     """
     def __init__(self, **options):
         Filter.__init__(self, **options)
@@ -303,7 +298,7 @@ class GobbleFilter(Filter):
         if left < len(value):
             return value[left:], 0
         else:
-            return '', left - len(value)
+            return u'', left - len(value)
 
     def filter(self, lexer, stream):
         n = self.n
@@ -314,18 +309,17 @@ class GobbleFilter(Filter):
             (parts[0], left) = self.gobble(parts[0], left)
             for i in range(1, len(parts)):
                 (parts[i], left) = self.gobble(parts[i], n)
-            value = '\n'.join(parts)
+            value = u'\n'.join(parts)
 
             if value != '':
                 yield ttype, value
 
 
 class TokenMergeFilter(Filter):
-    """
-    Merges consecutive tokens with the same token type in the output stream of a
-    lexer.
+    """Merges consecutive tokens with the same token type in the output
+    stream of a lexer.
 
-    *New in Pygments 1.2.*
+    .. versionadded:: 1.2
     """
     def __init__(self, **options):
         Filter.__init__(self, **options)

packages/wakatime/packages/pygments/formatter.py

@@ -5,20 +5,20 @@
 
     Base formatter class.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import codecs
 
-from pygments.util import get_bool_opt
+from pygments.util import get_bool_opt, string_types
 from pygments.styles import get_style_by_name
 
 __all__ = ['Formatter']
 
 
 def _lookup_style(style):
-    if isinstance(style, basestring):
+    if isinstance(style, string_types):
         return get_style_by_name(style)
     return style
 
@@ -65,13 +65,13 @@ class Formatter(object):
 
     def __init__(self, **options):
         self.style = _lookup_style(options.get('style', 'default'))
-        self.full  = get_bool_opt(options, 'full', False)
+        self.full = get_bool_opt(options, 'full', False)
         self.title = options.get('title', '')
         self.encoding = options.get('encoding', None) or None
-        if self.encoding == 'guess':
-            # can happen for pygmentize -O encoding=guess
+        if self.encoding in ('guess', 'chardet'):
+            # can happen for e.g. pygmentize -O encoding=guess
             self.encoding = 'utf-8'
-        self.encoding = options.get('outencoding', None) or self.encoding
+        self.encoding = options.get('outencoding') or self.encoding
         self.options = options
 
     def get_style_defs(self, arg=''):

packages/wakatime/packages/pygments/formatters/__init__.py

@@ -0,0 +1,153 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.formatters
+    ~~~~~~~~~~~~~~~~~~~
+
+    Pygments formatters.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+import sys
+import types
+import fnmatch
+from os.path import basename
+
+from pygments.formatters._mapping import FORMATTERS
+from pygments.plugin import find_plugin_formatters
+from pygments.util import ClassNotFound, itervalues
+
+__all__ = ['get_formatter_by_name', 'get_formatter_for_filename',
+           'get_all_formatters', 'load_formatter_from_file'] + list(FORMATTERS)
+
+_formatter_cache = {}  # classes by name
+_pattern_cache = {}
+
+
+def _fn_matches(fn, glob):
+    """Return whether the supplied file name fn matches pattern filename."""
+    if glob not in _pattern_cache:
+        pattern = _pattern_cache[glob] = re.compile(fnmatch.translate(glob))
+        return pattern.match(fn)
+    return _pattern_cache[glob].match(fn)
+
+
+def _load_formatters(module_name):
+    """Load a formatter (and all others in the module too)."""
+    mod = __import__(module_name, None, None, ['__all__'])
+    for formatter_name in mod.__all__:
+        cls = getattr(mod, formatter_name)
+        _formatter_cache[cls.name] = cls
+
+
+def get_all_formatters():
+    """Return a generator for all formatter classes."""
+    # NB: this returns formatter classes, not info like get_all_lexers().
+    for info in itervalues(FORMATTERS):
+        if info[1] not in _formatter_cache:
+            _load_formatters(info[0])
+        yield _formatter_cache[info[1]]
+    for _, formatter in find_plugin_formatters():
+        yield formatter
+
+
+def find_formatter_class(alias):
+    """Lookup a formatter by alias.
+
+    Returns None if not found.
+    """
+    for module_name, name, aliases, _, _ in itervalues(FORMATTERS):
+        if alias in aliases:
+            if name not in _formatter_cache:
+                _load_formatters(module_name)
+            return _formatter_cache[name]
+    for _, cls in find_plugin_formatters():
+        if alias in cls.aliases:
+            return cls
+
+
+def get_formatter_by_name(_alias, **options):
+    """Lookup and instantiate a formatter by alias.
+
+    Raises ClassNotFound if not found.
+    """
+    cls = find_formatter_class(_alias)
+    if cls is None:
+        raise ClassNotFound("no formatter found for name %r" % _alias)
+    return cls(**options)
+
+
+def load_formatter_from_file(filename, formattername="CustomFormatter",
+                             **options):
+    """Load a formatter from a file.
+
+    This method expects a file located relative to the current working
+    directory, which contains a class named CustomFormatter. By default,
+    it expects the Formatter to be named CustomFormatter; you can specify
+    your own class name as the second argument to this function.
+
+    Users should be very careful with the input, because this method
+    is equivalent to running eval on the input file.
+
+    Raises ClassNotFound if there are any problems importing the Formatter.
+
+    .. versionadded:: 2.2
+    """
+    try:
+        # This empty dict will contain the namespace for the exec'd file
+        custom_namespace = {}
+        exec(open(filename, 'rb').read(), custom_namespace)
+        # Retrieve the class `formattername` from that namespace
+        if formattername not in custom_namespace:
+            raise ClassNotFound('no valid %s class found in %s' %
+                                (formattername, filename))
+        formatter_class = custom_namespace[formattername]
+        # And finally instantiate it with the options
+        return formatter_class(**options)
+    except IOError as err:
+        raise ClassNotFound('cannot read %s' % filename)
+    except ClassNotFound as err:
+        raise
+    except Exception as err:
+        raise ClassNotFound('error when loading custom formatter: %s' % err)
+
+
+def get_formatter_for_filename(fn, **options):
+    """Lookup and instantiate a formatter by filename pattern.
+
+    Raises ClassNotFound if not found.
+    """
+    fn = basename(fn)
+    for modname, name, _, filenames, _ in itervalues(FORMATTERS):
+        for filename in filenames:
+            if _fn_matches(fn, filename):
+                if name not in _formatter_cache:
+                    _load_formatters(modname)
+                return _formatter_cache[name](**options)
+    for cls in find_plugin_formatters():
+        for filename in cls.filenames:
+            if _fn_matches(fn, filename):
+                return cls(**options)
+    raise ClassNotFound("no formatter found for file name %r" % fn)
+
+
+class _automodule(types.ModuleType):
+    """Automatically import formatters."""
+
+    def __getattr__(self, name):
+        info = FORMATTERS.get(name)
+        if info:
+            _load_formatters(info[0])
+            cls = _formatter_cache[info[1]]
+            setattr(self, name, cls)
+            return cls
+        raise AttributeError(name)
+
+
+oldmod = sys.modules[__name__]
+newmod = _automodule(__name__)
+newmod.__dict__.update(oldmod.__dict__)
+sys.modules[__name__] = newmod
+del newmod.newmod, newmod.oldmod, newmod.sys, newmod.types

packages/wakatime/packages/pygments/formatters/_mapping.py

@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.formatters._mapping
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Formatter mapping definitions. This file is generated by itself. Everytime
+    you change something on a builtin formatter definition, run this script from
+    the formatters folder to update it.
+
+    Do not alter the FORMATTERS dictionary by hand.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from __future__ import print_function
+
+FORMATTERS = {
+    'BBCodeFormatter': ('pygments.formatters.bbcode', 'BBCode', ('bbcode', 'bb'), (), 'Format tokens with BBcodes. These formatting codes are used by many bulletin boards, so you can highlight your sourcecode with pygments before posting it there.'),
+    'BmpImageFormatter': ('pygments.formatters.img', 'img_bmp', ('bmp', 'bitmap'), ('*.bmp',), 'Create a bitmap image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
+    'GifImageFormatter': ('pygments.formatters.img', 'img_gif', ('gif',), ('*.gif',), 'Create a GIF image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
+    'HtmlFormatter': ('pygments.formatters.html', 'HTML', ('html',), ('*.html', '*.htm'), "Format tokens as HTML 4 ``<span>`` tags within a ``<pre>`` tag, wrapped in a ``<div>`` tag. The ``<div>``'s CSS class can be set by the `cssclass` option."),
+    'IRCFormatter': ('pygments.formatters.irc', 'IRC', ('irc', 'IRC'), (), 'Format tokens with IRC color sequences'),
+    'ImageFormatter': ('pygments.formatters.img', 'img', ('img', 'IMG', 'png'), ('*.png',), 'Create a PNG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
+    'JpgImageFormatter': ('pygments.formatters.img', 'img_jpg', ('jpg', 'jpeg'), ('*.jpg',), 'Create a JPEG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
+    'LatexFormatter': ('pygments.formatters.latex', 'LaTeX', ('latex', 'tex'), ('*.tex',), 'Format tokens as LaTeX code. This needs the `fancyvrb` and `color` standard packages.'),
+    'NullFormatter': ('pygments.formatters.other', 'Text only', ('text', 'null'), ('*.txt',), 'Output the text unchanged without any formatting.'),
+    'RawTokenFormatter': ('pygments.formatters.other', 'Raw tokens', ('raw', 'tokens'), ('*.raw',), 'Format tokens as a raw representation for storing token streams.'),
+    'RtfFormatter': ('pygments.formatters.rtf', 'RTF', ('rtf',), ('*.rtf',), 'Format tokens as RTF markup. This formatter automatically outputs full RTF documents with color information and other useful stuff. Perfect for Copy and Paste into Microsoft(R) Word(R) documents.'),
+    'SvgFormatter': ('pygments.formatters.svg', 'SVG', ('svg',), ('*.svg',), 'Format tokens as an SVG graphics file.  This formatter is still experimental. Each line of code is a ``<text>`` element with explicit ``x`` and ``y`` coordinates containing ``<tspan>`` elements with the individual token styles.'),
+    'Terminal256Formatter': ('pygments.formatters.terminal256', 'Terminal256', ('terminal256', 'console256', '256'), (), 'Format tokens with ANSI color sequences, for output in a 256-color terminal or console.  Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'),
+    'TerminalFormatter': ('pygments.formatters.terminal', 'Terminal', ('terminal', 'console'), (), 'Format tokens with ANSI color sequences, for output in a text console. Color sequences are terminated at newlines, so that paging the output works correctly.'),
+    'TerminalTrueColorFormatter': ('pygments.formatters.terminal256', 'TerminalTrueColor', ('terminal16m', 'console16m', '16m'), (), 'Format tokens with ANSI color sequences, for output in a true-color terminal or console.  Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'),
+    'TestcaseFormatter': ('pygments.formatters.other', 'Testcase', ('testcase',), (), 'Format tokens as appropriate for a new testcase.')
+}
+
+if __name__ == '__main__':  # pragma: no cover
+    import sys
+    import os
+
+    # lookup formatters
+    found_formatters = []
+    imports = []
+    sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
+    from pygments.util import docstring_headline
+
+    for root, dirs, files in os.walk('.'):
+        for filename in files:
+            if filename.endswith('.py') and not filename.startswith('_'):
+                module_name = 'pygments.formatters%s.%s' % (
+                    root[1:].replace('/', '.'), filename[:-3])
+                print(module_name)
+                module = __import__(module_name, None, None, [''])
+                for formatter_name in module.__all__:
+                    formatter = getattr(module, formatter_name)
+                    found_formatters.append(
+                        '%r: %r' % (formatter_name,
+                                    (module_name,
+                                     formatter.name,
+                                     tuple(formatter.aliases),
+                                     tuple(formatter.filenames),
+                                     docstring_headline(formatter))))
+    # sort them to make the diff minimal
+    found_formatters.sort()
+
+    # extract useful sourcecode from this file
+    with open(__file__) as fp:
+        content = fp.read()
+        # replace crnl to nl for Windows.
+        #
+        # Note that, originally, contributers should keep nl of master
+        # repository, for example by using some kind of automatic
+        # management EOL, like `EolExtension
+        #  <https://www.mercurial-scm.org/wiki/EolExtension>`.
+        content = content.replace("\r\n", "\n")
+    header = content[:content.find('FORMATTERS = {')]
+    footer = content[content.find("if __name__ == '__main__':"):]
+
+    # write new file
+    with open(__file__, 'w') as fp:
+        fp.write(header)
+        fp.write('FORMATTERS = {\n    %s\n}\n\n' % ',\n    '.join(found_formatters))
+        fp.write(footer)
+
+    print ('=== %d formatters processed.' % len(found_formatters))

packages/wakatime/packages/pygments/formatters/bbcode.py

@@ -5,7 +5,7 @@
 
     BBcode formatter.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

packages/wakatime/packages/pygments/formatters/html.py

@@ -5,18 +5,20 @@
 
     Formatter for HTML output.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+from __future__ import print_function
+
 import os
 import sys
 import os.path
-import StringIO
 
 from pygments.formatter import Formatter
 from pygments.token import Token, Text, STANDARD_TYPES
-from pygments.util import get_bool_opt, get_int_opt, get_list_opt, bytes
+from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
+    StringIO, string_types, iteritems
 
 try:
     import ctags
@@ -34,21 +36,11 @@ _escape_html_table = {
     ord("'"): u''',
 }
 
+
 def escape_html(text, table=_escape_html_table):
     """Escape &, <, > as well as single and double quotes for HTML."""
     return text.translate(table)
 
-def get_random_id():
-    """Return a random id for javascript fields."""
-    from random import random
-    from time import time
-    try:
-        from hashlib import sha1 as sha
-    except ImportError:
-        import sha
-        sha = sha.new
-    return sha('%s|%s' % (random(), time())).hexdigest()
-
 
 def _get_ttype_class(ttype):
     fname = STANDARD_TYPES.get(ttype)
@@ -148,7 +140,7 @@ class HtmlFormatter(Formatter):
 
     When `tagsfile` is set to the path of a ctags index file, it is used to
     generate hyperlinks from names to their definition.  You must enable
-    `anchorlines` and run ctags with the `-n` option for this to work.  The
+    `lineanchors` and run ctags with the `-n` option for this to work.  The
     `python-ctags` module from PyPI must be installed to use this feature;
     otherwise a `RuntimeError` will be raised.
 
@@ -218,29 +210,34 @@ class HtmlFormatter(Formatter):
         If you set this option, the default selector for `get_style_defs()`
         will be this class.
 
-        *New in Pygments 0.9:* If you select the ``'table'`` line numbers, the
-        wrapping table will have a CSS class of this string plus ``'table'``,
-        the default is accordingly ``'highlighttable'``.
+        .. versionadded:: 0.9
+           If you select the ``'table'`` line numbers, the wrapping table will
+           have a CSS class of this string plus ``'table'``, the default is
+           accordingly ``'highlighttable'``.
 
     `cssstyles`
         Inline CSS styles for the wrapping ``<div>`` tag (default: ``''``).
 
     `prestyles`
-        Inline CSS styles for the ``<pre>`` tag (default: ``''``).  *New in
-        Pygments 0.11.*
+        Inline CSS styles for the ``<pre>`` tag (default: ``''``).
+
+        .. versionadded:: 0.11
 
     `cssfile`
         If the `full` option is true and this option is given, it must be the
         name of an external file. If the filename does not include an absolute
         path, the file's path will be assumed to be relative to the main output
         file's path, if the latter can be found. The stylesheet is then written
-        to this file instead of the HTML file. *New in Pygments 0.6.*
+        to this file instead of the HTML file.
+
+        .. versionadded:: 0.6
 
     `noclobber_cssfile`
         If `cssfile` is given and the specified file exists, the css file will
         not be overwritten. This allows the use of the `full` option in
         combination with a user specified css file. Default is ``False``.
-        *New in Pygments 1.1.*
+
+        .. versionadded:: 1.1
 
     `linenos`
         If set to ``'table'``, output line numbers as a table with two cells,
@@ -263,7 +260,9 @@ class HtmlFormatter(Formatter):
         125%``).
 
     `hl_lines`
-        Specify a list of lines to be highlighted.  *New in Pygments 0.11.*
+        Specify a list of lines to be highlighted.
+
+        .. versionadded:: 0.11
 
     `linenostart`
         The line number for the first line (default: ``1``).
@@ -279,24 +278,30 @@ class HtmlFormatter(Formatter):
         If set to ``True``, the formatter won't output the background color
         for the wrapping element (this automatically defaults to ``False``
         when there is no wrapping element [eg: no argument for the
-        `get_syntax_defs` method given]) (default: ``False``). *New in
-        Pygments 0.6.*
+        `get_syntax_defs` method given]) (default: ``False``).
+
+        .. versionadded:: 0.6
 
     `lineseparator`
         This string is output between lines of code. It defaults to ``"\n"``,
         which is enough to break a line inside ``<pre>`` tags, but you can
-        e.g. set it to ``"<br>"`` to get HTML line breaks. *New in Pygments
-        0.7.*
+        e.g. set it to ``"<br>"`` to get HTML line breaks.
+
+        .. versionadded:: 0.7
 
     `lineanchors`
         If set to a nonempty string, e.g. ``foo``, the formatter will wrap each
         output line in an anchor tag with a ``name`` of ``foo-linenumber``.
-        This allows easy linking to certain lines. *New in Pygments 0.9.*
+        This allows easy linking to certain lines.
+
+        .. versionadded:: 0.9
 
     `linespans`
         If set to a nonempty string, e.g. ``foo``, the formatter will wrap each
         output line in a span tag with an ``id`` of ``foo-linenumber``.
-        This allows easy access to lines via javascript. *New in Pygments 1.6.*
+        This allows easy access to lines via javascript.
+
+        .. versionadded:: 1.6
 
     `anchorlinenos`
         If set to `True`, will wrap line numbers in <a> tags. Used in
@@ -306,18 +311,26 @@ class HtmlFormatter(Formatter):
         If set to the path of a ctags file, wrap names in anchor tags that
         link to their definitions. `lineanchors` should be used, and the
         tags file should specify line numbers (see the `-n` option to ctags).
-        *New in Pygments 1.6.*
+
+        .. versionadded:: 1.6
 
     `tagurlformat`
         A string formatting pattern used to generate links to ctags definitions.
         Available variables are `%(path)s`, `%(fname)s` and `%(fext)s`.
         Defaults to an empty string, resulting in just `#prefix-number` links.
-        *New in Pygments 1.6.*
+
+        .. versionadded:: 1.6
+
+    `filename`
+        A string used to generate a filename when rendering <pre> blocks,
+        for example if displaying source code.
+
+        .. versionadded:: 2.1
 
 
     **Subclassing the HTML formatter**
 
-    *New in Pygments 0.7.*
+    .. versionadded:: 0.7
 
     The HTML formatter is now built in a way that allows easy subclassing, thus
     customizing the output HTML code. The `format()` method calls
@@ -381,6 +394,7 @@ class HtmlFormatter(Formatter):
         self.noclobber_cssfile = get_bool_opt(options, 'noclobber_cssfile', False)
         self.tagsfile = self._decodeifneeded(options.get('tagsfile', ''))
         self.tagurlformat = self._decodeifneeded(options.get('tagurlformat', ''))
+        self.filename = self._decodeifneeded(options.get('filename', ''))
 
         if self.tagsfile:
             if not ctags:
@@ -421,6 +435,15 @@ class HtmlFormatter(Formatter):
             return self.classprefix + ttypeclass
         return ''
 
+    def _get_css_classes(self, ttype):
+        """Return the css classes of this token type prefixed with
+        the classprefix option."""
+        cls = self._get_css_class(ttype)
+        while ttype not in STANDARD_TYPES:
+            ttype = ttype.parent
+            cls = self._get_css_class(ttype) + ' ' + cls
+        return cls
+
     def _create_stylesheet(self):
         t2c = self.ttype2class = {Token: ''}
         c2s = self.class2style = {}
@@ -453,7 +476,7 @@ class HtmlFormatter(Formatter):
         """
         if arg is None:
             arg = ('cssclass' in self.options and '.'+self.cssclass or '')
-        if isinstance(arg, basestring):
+        if isinstance(arg, string_types):
             args = [arg]
         else:
             args = list(arg)
@@ -467,7 +490,7 @@ class HtmlFormatter(Formatter):
             return ', '.join(tmp)
 
         styles = [(level, ttype, cls, style)
-                  for cls, (style, ttype, level) in self.class2style.iteritems()
+                  for cls, (style, ttype, level) in iteritems(self.class2style)
                   if cls and style]
         styles.sort()
         lines = ['%s { %s } /* %s */' % (prefix(cls), style, repr(ttype)[6:])
@@ -505,36 +528,37 @@ class HtmlFormatter(Formatter):
                     cssfilename = os.path.join(os.path.dirname(filename),
                                                self.cssfile)
                 except AttributeError:
-                    print >>sys.stderr, 'Note: Cannot determine output file name, ' \
-                          'using current directory as base for the CSS file name'
+                    print('Note: Cannot determine output file name, '
+                          'using current directory as base for the CSS file name',
+                          file=sys.stderr)
                     cssfilename = self.cssfile
             # write CSS file only if noclobber_cssfile isn't given as an option.
             try:
                 if not os.path.exists(cssfilename) or not self.noclobber_cssfile:
                     cf = open(cssfilename, "w")
                     cf.write(CSSFILE_TEMPLATE %
-                            {'styledefs': self.get_style_defs('body')})
+                             {'styledefs': self.get_style_defs('body')})
                     cf.close()
-            except IOError, err:
+            except IOError as err:
                 err.strerror = 'Error writing CSS file: ' + err.strerror
                 raise
 
             yield 0, (DOC_HEADER_EXTERNALCSS %
-                      dict(title     = self.title,
-                           cssfile   = self.cssfile,
-                           encoding  = self.encoding))
+                      dict(title=self.title,
+                           cssfile=self.cssfile,
+                           encoding=self.encoding))
         else:
             yield 0, (DOC_HEADER %
-                      dict(title     = self.title,
-                           styledefs = self.get_style_defs('body'),
-                           encoding  = self.encoding))
+                      dict(title=self.title,
+                           styledefs=self.get_style_defs('body'),
+                           encoding=self.encoding))
 
         for t, line in inner:
             yield t, line
         yield 0, DOC_FOOTER
 
     def _wrap_tablelinenos(self, inner):
-        dummyoutfile = StringIO.StringIO()
+        dummyoutfile = StringIO()
         lncount = 0
         for t, line in inner:
             if t:
@@ -606,35 +630,35 @@ class HtmlFormatter(Formatter):
         if self.noclasses:
             if sp:
                 for t, line in lines:
-                    if num%sp == 0:
+                    if num % sp == 0:
                         style = 'background-color: #ffffc0; padding: 0 5px 0 5px'
                     else:
                         style = 'background-color: #f0f0f0; padding: 0 5px 0 5px'
-                    yield 1, '<span style="%s">%*s</span> ' % (
-                        style, mw, (num%st and ' ' or num)) + line
+                    yield 1, '<span style="%s">%*s </span>' % (
+                        style, mw, (num % st and ' ' or num)) + line
                     num += 1
             else:
                 for t, line in lines:
                     yield 1, ('<span style="background-color: #f0f0f0; '
-                              'padding: 0 5px 0 5px">%*s</span> ' % (
-                              mw, (num%st and ' ' or num)) + line)
+                              'padding: 0 5px 0 5px">%*s </span>' % (
+                                  mw, (num % st and ' ' or num)) + line)
                     num += 1
         elif sp:
             for t, line in lines:
-                yield 1, '<span class="lineno%s">%*s</span> ' % (
-                    num%sp == 0 and ' special' or '', mw,
-                    (num%st and ' ' or num)) + line
+                yield 1, '<span class="lineno%s">%*s </span>' % (
+                    num % sp == 0 and ' special' or '', mw,
+                    (num % st and ' ' or num)) + line
                 num += 1
         else:
             for t, line in lines:
-                yield 1, '<span class="lineno">%*s</span> ' % (
-                    mw, (num%st and ' ' or num)) + line
+                yield 1, '<span class="lineno">%*s </span>' % (
+                    mw, (num % st and ' ' or num)) + line
                 num += 1
 
     def _wrap_lineanchors(self, inner):
         s = self.lineanchors
-        i = self.linenostart - 1 # subtract 1 since we have to increment i
-                                 # *before* yielding
+        # subtract 1 since we have to increment i *before* yielding
+        i = self.linenostart - 1
         for t, line in inner:
             if t:
                 i += 1
@@ -655,14 +679,14 @@ class HtmlFormatter(Formatter):
     def _wrap_div(self, inner):
         style = []
         if (self.noclasses and not self.nobackground and
-            self.style.background_color is not None):
+                self.style.background_color is not None):
             style.append('background: %s' % (self.style.background_color,))
         if self.cssstyles:
             style.append(self.cssstyles)
         style = '; '.join(style)
 
-        yield 0, ('<div' + (self.cssclass and ' class="%s"' % self.cssclass)
-                  + (style and (' style="%s"' % style)) + '>')
+        yield 0, ('<div' + (self.cssclass and ' class="%s"' % self.cssclass) +
+                  (style and (' style="%s"' % style)) + '>')
         for tup in inner:
             yield tup
         yield 0, '</div>\n'
@@ -675,7 +699,12 @@ class HtmlFormatter(Formatter):
             style.append('line-height: 125%')
         style = '; '.join(style)
 
-        yield 0, ('<pre' + (style and ' style="%s"' % style) + '>')
+        if self.filename:
+            yield 0, ('<span class="filename">' + self.filename + '</span>')
+
+        # the empty span here is to keep leading empty lines from being
+        # ignored by HTML parsers
+        yield 0, ('<pre' + (style and ' style="%s"' % style) + '><span></span>')
         for tup in inner:
             yield tup
         yield 0, '</pre>'
@@ -694,7 +723,7 @@ class HtmlFormatter(Formatter):
         tagsfile = self.tagsfile
 
         lspan = ''
-        line = ''
+        line = []
         for ttype, value in tokensource:
             if nocls:
                 cclass = getcls(ttype)
@@ -703,7 +732,7 @@ class HtmlFormatter(Formatter):
                     cclass = getcls(ttype)
                 cspan = cclass and '<span style="%s">' % c2s[cclass][0] or ''
             else:
-                cls = self._get_css_class(ttype)
+                cls = self._get_css_classes(ttype)
                 cspan = cls and '<span class="%s">' % cls or ''
 
             parts = value.translate(escape_table).split('\n')
@@ -725,30 +754,31 @@ class HtmlFormatter(Formatter):
             for part in parts[:-1]:
                 if line:
                     if lspan != cspan:
-                        line += (lspan and '</span>') + cspan + part + \
-                                (cspan and '</span>') + lsep
-                    else: # both are the same
-                        line += part + (lspan and '</span>') + lsep
-                    yield 1, line
-                    line = ''
+                        line.extend(((lspan and '</span>'), cspan, part,
+                                     (cspan and '</span>'), lsep))
+                    else:  # both are the same
+                        line.extend((part, (lspan and '</span>'), lsep))
+                    yield 1, ''.join(line)
+                    line = []
                 elif part:
-                    yield 1, cspan + part + (cspan and '</span>') + lsep
+                    yield 1, ''.join((cspan, part, (cspan and '</span>'), lsep))
                 else:
                     yield 1, lsep
             # for the last line
             if line and parts[-1]:
                 if lspan != cspan:
-                    line += (lspan and '</span>') + cspan + parts[-1]
+                    line.extend(((lspan and '</span>'), cspan, parts[-1]))
                     lspan = cspan
                 else:
-                    line += parts[-1]
+                    line.append(parts[-1])
             elif parts[-1]:
-                line = cspan + parts[-1]
+                line = [cspan, parts[-1]]
                 lspan = cspan
             # else we neither have to open a new span nor set lspan
 
         if line:
-            yield 1, line + (lspan and '</span>') + lsep
+            line.extend(((lspan and '</span>'), lsep))
+            yield 1, ''.join(line)
 
     def _lookup_ctag(self, token):
         entry = ctags.TagEntry()
@@ -767,7 +797,7 @@ class HtmlFormatter(Formatter):
         for i, (t, value) in enumerate(tokensource):
             if t != 1:
                 yield t, value
-            if i + 1 in hls: # i + 1 because Python indexes start at 0
+            if i + 1 in hls:  # i + 1 because Python indexes start at 0
                 if self.noclasses:
                     style = ''
                     if self.style.highlight_color is not None:

packages/wakatime/packages/pygments/formatters/img.py

@@ -5,15 +5,18 @@
 
     Formatter for Pixmap output.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+import os
 import sys
 
 from pygments.formatter import Formatter
-from pygments.util import get_bool_opt, get_int_opt, \
-     get_list_opt, get_choice_opt
+from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
+    get_choice_opt, xrange
+
+import subprocess
 
 # Import this carefully
 try:
@@ -25,7 +28,10 @@ except ImportError:
 try:
     import _winreg
 except ImportError:
-    _winreg = None
+    try:
+        import winreg as _winreg
+    except ImportError:
+        _winreg = None
 
 __all__ = ['ImageFormatter', 'GifImageFormatter', 'JpgImageFormatter',
            'BmpImageFormatter']
@@ -42,6 +48,7 @@ STYLES = {
 # A sane default for modern systems
 DEFAULT_FONT_NAME_NIX = 'Bitstream Vera Sans Mono'
 DEFAULT_FONT_NAME_WIN = 'Courier New'
+DEFAULT_FONT_NAME_MAC = 'Courier New'
 
 
 class PilNotAvailable(ImportError):
@@ -66,20 +73,28 @@ class FontManager(object):
             if not font_name:
                 self.font_name = DEFAULT_FONT_NAME_WIN
             self._create_win()
+        elif sys.platform.startswith('darwin'):
+            if not font_name:
+                self.font_name = DEFAULT_FONT_NAME_MAC
+            self._create_mac()
         else:
             if not font_name:
                 self.font_name = DEFAULT_FONT_NAME_NIX
             self._create_nix()
 
     def _get_nix_font_path(self, name, style):
-        from commands import getstatusoutput
-        exit, out = getstatusoutput('fc-list "%s:style=%s" file' %
-                                    (name, style))
-        if not exit:
-            lines = out.splitlines()
-            if lines:
-                path = lines[0].strip().strip(':')
-                return path
+        proc = subprocess.Popen(['fc-list', "%s:style=%s" % (name, style), 'file'],
+                                stdout=subprocess.PIPE, stderr=None)
+        stdout, _ = proc.communicate()
+        if proc.returncode == 0:
+            lines = stdout.splitlines()
+            for line in lines:
+                if line.startswith(b'Fontconfig warning:'):
+                    continue
+                path = line.decode().strip().strip(':')
+                if path:
+                    return path
+            return None
 
     def _create_nix(self):
         for name in STYLES['NORMAL']:
@@ -102,6 +117,37 @@ class FontManager(object):
                 else:
                     self.fonts[style] = self.fonts['NORMAL']
 
+    def _get_mac_font_path(self, font_map, name, style):
+        return font_map.get((name + ' ' + style).strip().lower())
+
+    def _create_mac(self):
+        font_map = {}
+        for font_dir in (os.path.join(os.getenv("HOME"), 'Library/Fonts/'),
+                         '/Library/Fonts/', '/System/Library/Fonts/'):
+            font_map.update(
+                ((os.path.splitext(f)[0].lower(), os.path.join(font_dir, f))
+                    for f in os.listdir(font_dir) if f.lower().endswith('ttf')))
+
+        for name in STYLES['NORMAL']:
+            path = self._get_mac_font_path(font_map, self.font_name, name)
+            if path is not None:
+                self.fonts['NORMAL'] = ImageFont.truetype(path, self.font_size)
+                break
+        else:
+            raise FontNotFound('No usable fonts named: "%s"' %
+                               self.font_name)
+        for style in ('ITALIC', 'BOLD', 'BOLDITALIC'):
+            for stylename in STYLES[style]:
+                path = self._get_mac_font_path(font_map, self.font_name, stylename)
+                if path is not None:
+                    self.fonts[style] = ImageFont.truetype(path, self.font_size)
+                    break
+            else:
+                if style == 'BOLDITALIC':
+                    self.fonts[style] = self.fonts['BOLD']
+                else:
+                    self.fonts[style] = self.fonts['NORMAL']
+
     def _lookup_win(self, key, basename, styles, fail=False):
         for suffix in ('', ' (TrueType)'):
             for style in styles:
@@ -169,7 +215,7 @@ class ImageFormatter(Formatter):
     Create a PNG image from source code. This uses the Python Imaging Library to
     generate a pixmap from the source code.
 
-    *New in Pygments 0.10.*
+    .. versionadded:: 0.10
 
     Additional options accepted:
 
@@ -191,7 +237,7 @@ class ImageFormatter(Formatter):
         bold and italic fonts will be generated.  This really should be a
         monospace font to look sane.
 
-        Default: "Bitstream Vera Sans Mono"
+        Default: "Bitstream Vera Sans Mono" on Windows, Courier New on \*nix
 
     `font_size`
         The font size in points to be used.
@@ -258,12 +304,16 @@ class ImageFormatter(Formatter):
         Default: 6
 
     `hl_lines`
-        Specify a list of lines to be highlighted.  *New in Pygments 1.2.*
+        Specify a list of lines to be highlighted.
+
+        .. versionadded:: 1.2
 
         Default: empty list
 
     `hl_color`
-        Specify the color for highlighting lines.  *New in Pygments 1.2.*
+        Specify the color for highlighting lines.
+
+        .. versionadded:: 1.2
 
         Default: highlight color of the selected style
     """
@@ -285,6 +335,7 @@ class ImageFormatter(Formatter):
             raise PilNotAvailable(
                 'Python Imaging Library is required for this formatter')
         Formatter.__init__(self, **options)
+        self.encoding = 'latin1'  # let pygments.format() do the right thing
         # Read the style
         self.styles = dict(self.style)
         if self.style.background_color is None:
@@ -305,20 +356,20 @@ class ImageFormatter(Formatter):
         self.line_number_fg = options.get('line_number_fg', '#886')
         self.line_number_bg = options.get('line_number_bg', '#eed')
         self.line_number_chars = get_int_opt(options,
-                                        'line_number_chars', 2)
+                                             'line_number_chars', 2)
         self.line_number_bold = get_bool_opt(options,
-                                        'line_number_bold', False)
+                                             'line_number_bold', False)
         self.line_number_italic = get_bool_opt(options,
-                                        'line_number_italic', False)
+                                               'line_number_italic', False)
         self.line_number_pad = get_int_opt(options, 'line_number_pad', 6)
         self.line_numbers = get_bool_opt(options, 'line_numbers', True)
         self.line_number_separator = get_bool_opt(options,
-                                        'line_number_separator', True)
+                                                  'line_number_separator', True)
         self.line_number_step = get_int_opt(options, 'line_number_step', 1)
         self.line_number_start = get_int_opt(options, 'line_number_start', 1)
         if self.line_numbers:
             self.line_number_width = (self.fontw * self.line_number_chars +
-                                   self.line_number_pad * 2)
+                                      self.line_number_pad * 2)
         else:
             self.line_number_width = 0
         self.hl_lines = []
@@ -427,7 +478,7 @@ class ImageFormatter(Formatter):
             # quite complex.
             value = value.expandtabs(4)
             lines = value.splitlines(True)
-            #print lines
+            # print lines
             for i, line in enumerate(lines):
                 temp = line.rstrip('\n')
                 if temp:
@@ -468,9 +519,8 @@ class ImageFormatter(Formatter):
         draw = ImageDraw.Draw(im)
         recth = im.size[-1]
         rectw = self.image_pad + self.line_number_width - self.line_number_pad
-        draw.rectangle([(0, 0),
-                        (rectw, recth)],
-             fill=self.line_number_bg)
+        draw.rectangle([(0, 0), (rectw, recth)],
+                       fill=self.line_number_bg)
         draw.line([(rectw, 0), (rectw, recth)], fill=self.line_number_fg)
         del draw
 
@@ -513,8 +563,7 @@ class GifImageFormatter(ImageFormatter):
     Create a GIF image from source code. This uses the Python Imaging Library to
     generate a pixmap from the source code.
 
-    *New in Pygments 1.0.* (You could create GIF images before by passing a
-    suitable `image_format` option to the `ImageFormatter`.)
+    .. versionadded:: 1.0
     """
 
     name = 'img_gif'
@@ -528,8 +577,7 @@ class JpgImageFormatter(ImageFormatter):
     Create a JPEG image from source code. This uses the Python Imaging Library to
     generate a pixmap from the source code.
 
-    *New in Pygments 1.0.* (You could create JPEG images before by passing a
-    suitable `image_format` option to the `ImageFormatter`.)
+    .. versionadded:: 1.0
     """
 
     name = 'img_jpg'
@@ -543,8 +591,7 @@ class BmpImageFormatter(ImageFormatter):
     Create a bitmap image from source code. This uses the Python Imaging Library to
     generate a pixmap from the source code.
 
-    *New in Pygments 1.0.* (You could create bitmap images before by passing a
-    suitable `image_format` option to the `ImageFormatter`.)
+    .. versionadded:: 1.0
     """
 
     name = 'img_bmp'

packages/wakatime/packages/pygments/formatters/irc.py

@@ -0,0 +1,182 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.formatters.irc
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Formatter for IRC output
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import sys
+
+from pygments.formatter import Formatter
+from pygments.token import Keyword, Name, Comment, String, Error, \
+    Number, Operator, Generic, Token, Whitespace
+from pygments.util import get_choice_opt
+
+
+__all__ = ['IRCFormatter']
+
+
+#: Map token types to a tuple of color values for light and dark
+#: backgrounds.
+IRC_COLORS = {
+    Token:              ('',            ''),
+
+    Whitespace:         ('lightgray',   'darkgray'),
+    Comment:            ('lightgray',   'darkgray'),
+    Comment.Preproc:    ('teal',        'turquoise'),
+    Keyword:            ('darkblue',    'blue'),
+    Keyword.Type:       ('teal',        'turquoise'),
+    Operator.Word:      ('purple',      'fuchsia'),
+    Name.Builtin:       ('teal',        'turquoise'),
+    Name.Function:      ('darkgreen',   'green'),
+    Name.Namespace:     ('_teal_',      '_turquoise_'),
+    Name.Class:         ('_darkgreen_', '_green_'),
+    Name.Exception:     ('teal',        'turquoise'),
+    Name.Decorator:     ('darkgray',    'lightgray'),
+    Name.Variable:      ('darkred',     'red'),
+    Name.Constant:      ('darkred',     'red'),
+    Name.Attribute:     ('teal',        'turquoise'),
+    Name.Tag:           ('blue',        'blue'),
+    String:             ('brown',       'brown'),
+    Number:             ('darkblue',    'blue'),
+
+    Generic.Deleted:    ('red',        'red'),
+    Generic.Inserted:   ('darkgreen',  'green'),
+    Generic.Heading:    ('**',         '**'),
+    Generic.Subheading: ('*purple*',   '*fuchsia*'),
+    Generic.Error:      ('red',        'red'),
+
+    Error:              ('_red_',      '_red_'),
+}
+
+
+IRC_COLOR_MAP = {
+    'white': 0,
+    'black': 1,
+    'darkblue': 2,
+    'green': 3,
+    'red': 4,
+    'brown': 5,
+    'purple': 6,
+    'orange': 7,
+    'darkgreen': 7, #compat w/ ansi
+    'yellow': 8,
+    'lightgreen': 9,
+    'turquoise': 9, # compat w/ ansi
+    'teal': 10,
+    'lightblue': 11,
+    'darkred': 11, # compat w/ ansi
+    'blue': 12,
+    'fuchsia': 13,
+    'darkgray': 14,
+    'lightgray': 15,
+}
+
+def ircformat(color, text):
+    if len(color) < 1:
+        return text
+    add = sub = ''
+    if '_' in color: # italic
+        add += '\x1D'
+        sub = '\x1D' + sub
+        color = color.strip('_')
+    if '*' in color: # bold
+        add += '\x02'
+        sub = '\x02' + sub
+        color = color.strip('*')
+    # underline (\x1F) not supported
+    # backgrounds (\x03FF,BB) not supported
+    if len(color) > 0: # actual color - may have issues with ircformat("red", "blah")+"10" type stuff
+        add += '\x03' + str(IRC_COLOR_MAP[color]).zfill(2)
+        sub = '\x03' + sub
+    return add + text + sub
+    return '<'+add+'>'+text+'</'+sub+'>'
+
+
+class IRCFormatter(Formatter):
+    r"""
+    Format tokens with IRC color sequences
+
+    The `get_style_defs()` method doesn't do anything special since there is
+    no support for common styles.
+
+    Options accepted:
+
+    `bg`
+        Set to ``"light"`` or ``"dark"`` depending on the terminal's background
+        (default: ``"light"``).
+
+    `colorscheme`
+        A dictionary mapping token types to (lightbg, darkbg) color names or
+        ``None`` (default: ``None`` = use builtin colorscheme).
+
+    `linenos`
+        Set to ``True`` to have line numbers in the output as well
+        (default: ``False`` = no line numbers).
+    """
+    name = 'IRC'
+    aliases = ['irc', 'IRC']
+    filenames = []
+
+    def __init__(self, **options):
+        Formatter.__init__(self, **options)
+        self.darkbg = get_choice_opt(options, 'bg',
+                                     ['light', 'dark'], 'light') == 'dark'
+        self.colorscheme = options.get('colorscheme', None) or IRC_COLORS
+        self.linenos = options.get('linenos', False)
+        self._lineno = 0
+
+    def _write_lineno(self, outfile):
+        self._lineno += 1
+        outfile.write("\n%04d: " % self._lineno)
+
+    def _format_unencoded_with_lineno(self, tokensource, outfile):
+        self._write_lineno(outfile)
+
+        for ttype, value in tokensource:
+            if value.endswith("\n"):
+                self._write_lineno(outfile)
+                value = value[:-1]
+            color = self.colorscheme.get(ttype)
+            while color is None:
+                ttype = ttype[:-1]
+                color = self.colorscheme.get(ttype)
+            if color:
+                color = color[self.darkbg]
+                spl = value.split('\n')
+                for line in spl[:-1]:
+                    self._write_lineno(outfile)
+                    if line:
+                        outfile.write(ircformat(color, line[:-1]))
+                if spl[-1]:
+                    outfile.write(ircformat(color, spl[-1]))
+            else:
+                outfile.write(value)
+
+        outfile.write("\n")
+
+    def format_unencoded(self, tokensource, outfile):
+        if self.linenos:
+            self._format_unencoded_with_lineno(tokensource, outfile)
+            return
+
+        for ttype, value in tokensource:
+            color = self.colorscheme.get(ttype)
+            while color is None:
+                ttype = ttype[:-1]
+                color = self.colorscheme.get(ttype)
+            if color:
+                color = color[self.darkbg]
+                spl = value.split('\n')
+                for line in spl[:-1]:
+                    if line:
+                        outfile.write(ircformat(color, line))
+                    outfile.write('\n')
+                if spl[-1]:
+                    outfile.write(ircformat(color, spl[-1]))
+            else:
+                outfile.write(value)

packages/wakatime/packages/pygments/formatters/latex.py

@@ -5,13 +5,17 @@
 
     Formatter for LaTeX fancyvrb output.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+from __future__ import division
+
 from pygments.formatter import Formatter
+from pygments.lexer import Lexer
 from pygments.token import Token, STANDARD_TYPES
-from pygments.util import get_bool_opt, get_int_opt, StringIO
+from pygments.util import get_bool_opt, get_int_opt, StringIO, xrange, \
+    iteritems
 
 
 __all__ = ['LatexFormatter']
@@ -152,7 +156,7 @@ class LatexFormatter(Formatter):
 
     .. sourcecode:: latex
 
-        \begin{Verbatim}[commandchars=\\{\}]
+        \begin{Verbatim}[commandchars=\\\{\}]
         \PY{k}{def }\PY{n+nf}{foo}(\PY{n}{bar}):
             \PY{k}{pass}
         \end{Verbatim}
@@ -205,19 +209,40 @@ class LatexFormatter(Formatter):
     `commandprefix`
         The LaTeX commands used to produce colored output are constructed
         using this prefix and some letters (default: ``'PY'``).
-        *New in Pygments 0.7.*
 
-        *New in Pygments 0.10:* the default is now ``'PY'`` instead of ``'C'``.
+        .. versionadded:: 0.7
+        .. versionchanged:: 0.10
+           The default is now ``'PY'`` instead of ``'C'``.
 
     `texcomments`
         If set to ``True``, enables LaTeX comment lines.  That is, LaTex markup
         in comment tokens is not escaped so that LaTeX can render it (default:
-        ``False``).  *New in Pygments 1.2.*
+        ``False``).
+
+        .. versionadded:: 1.2
 
     `mathescape`
         If set to ``True``, enables LaTeX math mode escape in comments. That
         is, ``'$...$'`` inside a comment will trigger math mode (default:
-        ``False``).  *New in Pygments 1.2.*
+        ``False``).
+
+        .. versionadded:: 1.2
+
+    `escapeinside`
+        If set to a string of length 2, enables escaping to LaTeX. Text
+        delimited by these 2 characters is read as LaTeX code and
+        typeset accordingly. It has no effect in string literals. It has
+        no effect in comments if `texcomments` or `mathescape` is
+        set. (default: ``''``).
+
+        .. versionadded:: 2.0
+
+    `envname`
+        Allows you to pick an alternative environment name replacing Verbatim.
+        The alternate environment still has to support Verbatim's option syntax.
+        (default: ``'Verbatim'``).
+
+        .. versionadded:: 2.0
     """
     name = 'LaTeX'
     aliases = ['latex', 'tex']
@@ -235,10 +260,16 @@ class LatexFormatter(Formatter):
         self.commandprefix = options.get('commandprefix', 'PY')
         self.texcomments = get_bool_opt(options, 'texcomments', False)
         self.mathescape = get_bool_opt(options, 'mathescape', False)
+        self.escapeinside = options.get('escapeinside', '')
+        if len(self.escapeinside) == 2:
+            self.left = self.escapeinside[0]
+            self.right = self.escapeinside[1]
+        else:
+            self.escapeinside = ''
+        self.envname = options.get('envname', u'Verbatim')
 
         self._create_stylesheet()
 
-
     def _create_stylesheet(self):
         t2n = self.ttype2name = {Token: ''}
         c2d = self.cmd2def = {}
@@ -246,7 +277,7 @@ class LatexFormatter(Formatter):
 
         def rgbcolor(col):
             if col:
-                return ','.join(['%.2f' %(int(col[i] + col[i + 1], 16) / 255.0)
+                return ','.join(['%.2f' % (int(col[i] + col[i + 1], 16) / 255.0)
                                  for i in (0, 2, 4)])
             else:
                 return '1,1,1'
@@ -291,7 +322,7 @@ class LatexFormatter(Formatter):
         """
         cp = self.commandprefix
         styles = []
-        for name, definition in self.cmd2def.iteritems():
+        for name, definition in iteritems(self.cmd2def):
             styles.append(r'\expandafter\def\csname %s@tok@%s\endcsname{%s}' %
                           (cp, name, definition))
         return STYLE_TEMPLATE % {'cp': self.commandprefix,
@@ -306,14 +337,14 @@ class LatexFormatter(Formatter):
             realoutfile = outfile
             outfile = StringIO()
 
-        outfile.write(ur'\begin{Verbatim}[commandchars=\\\{\}')
+        outfile.write(u'\\begin{' + self.envname + u'}[commandchars=\\\\\\{\\}')
         if self.linenos:
             start, step = self.linenostart, self.linenostep
             outfile.write(u',numbers=left' +
                           (start and u',firstnumber=%d' % start or u'') +
                           (step and u',stepnumber=%d' % step or u''))
-        if self.mathescape or self.texcomments:
-            outfile.write(ur',codes={\catcode`\$=3\catcode`\^=7\catcode`\_=8}')
+        if self.mathescape or self.texcomments or self.escapeinside:
+            outfile.write(u',codes={\\catcode`\\$=3\\catcode`\\^=7\\catcode`\\_=8}')
         if self.verboptions:
             outfile.write(u',' + self.verboptions)
         outfile.write(u']\n')
@@ -329,7 +360,7 @@ class LatexFormatter(Formatter):
                         start += value[i]
 
                     value = value[len(start):]
-                    start = escape_tex(start, self.commandprefix)
+                    start = escape_tex(start, cp)
 
                     # ... but do not escape inside comment.
                     value = start + value
@@ -339,13 +370,26 @@ class LatexFormatter(Formatter):
                     in_math = False
                     for i, part in enumerate(parts):
                         if not in_math:
-                            parts[i] = escape_tex(part, self.commandprefix)
+                            parts[i] = escape_tex(part, cp)
                         in_math = not in_math
                     value = '$'.join(parts)
+                elif self.escapeinside:
+                    text = value
+                    value = ''
+                    while text:
+                        a, sep1, text = text.partition(self.left)
+                        if sep1:
+                            b, sep2, text = text.partition(self.right)
+                            if sep2:
+                                value += escape_tex(a, cp) + b
+                            else:
+                                value += escape_tex(a + sep1 + b, cp)
+                        else:
+                            value += escape_tex(a, cp)
                 else:
-                    value = escape_tex(value, self.commandprefix)
-            else:
-                value = escape_tex(value, self.commandprefix)
+                    value = escape_tex(value, cp)
+            elif ttype not in Token.Escape:
+                value = escape_tex(value, cp)
             styles = []
             while ttype is not Token:
                 try:
@@ -366,13 +410,73 @@ class LatexFormatter(Formatter):
             else:
                 outfile.write(value)
 
-        outfile.write(u'\\end{Verbatim}\n')
+        outfile.write(u'\\end{' + self.envname + u'}\n')
 
         if self.full:
+            encoding = self.encoding or 'utf8'
+            # map known existings encodings from LaTeX distribution
+            encoding = {
+                'utf_8': 'utf8',
+                'latin_1': 'latin1',
+                'iso_8859_1': 'latin1',
+            }.get(encoding.replace('-', '_'), encoding)
             realoutfile.write(DOC_TEMPLATE %
                 dict(docclass  = self.docclass,
                      preamble  = self.preamble,
                      title     = self.title,
-                     encoding  = self.encoding or 'latin1',
+                     encoding  = encoding,
                      styledefs = self.get_style_defs(),
                      code      = outfile.getvalue()))
+
+
+class LatexEmbeddedLexer(Lexer):
+    """
+    This lexer takes one lexer as argument, the lexer for the language
+    being formatted, and the left and right delimiters for escaped text.
+
+    First everything is scanned using the language lexer to obtain
+    strings and comments. All other consecutive tokens are merged and
+    the resulting text is scanned for escaped segments, which are given
+    the Token.Escape type. Finally text that is not escaped is scanned
+    again with the language lexer.
+    """
+    def __init__(self, left, right, lang, **options):
+        self.left = left
+        self.right = right
+        self.lang = lang
+        Lexer.__init__(self, **options)
+
+    def get_tokens_unprocessed(self, text):
+        buf = ''
+        idx = 0
+        for i, t, v in self.lang.get_tokens_unprocessed(text):
+            if t in Token.Comment or t in Token.String:
+                if buf:
+                    for x in self.get_tokens_aux(idx, buf):
+                        yield x
+                    buf = ''
+                yield i, t, v
+            else:
+                if not buf:
+                    idx = i
+                buf += v
+        if buf:
+            for x in self.get_tokens_aux(idx, buf):
+                yield x
+
+    def get_tokens_aux(self, index, text):
+        while text:
+            a, sep1, text = text.partition(self.left)
+            if a:
+                for i, t, v in self.lang.get_tokens_unprocessed(a):
+                    yield index + i, t, v
+                    index += len(a)
+            if sep1:
+                b, sep2, text = text.partition(self.right)
+                if sep2:
+                    yield index + len(sep1), Token.Escape, b
+                    index += len(sep1) + len(b) + len(sep2)
+                else:
+                    yield index, Token.Error, sep1
+                    index += len(sep1)
+                    text = b

packages/wakatime/packages/pygments/formatters/other.py

@@ -5,16 +5,16 @@
 
     Other formatters: NullFormatter, RawTokenFormatter.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 from pygments.formatter import Formatter
-from pygments.util import OptionError, get_choice_opt, b
+from pygments.util import OptionError, get_choice_opt
 from pygments.token import Token
 from pygments.console import colorize
 
-__all__ = ['NullFormatter', 'RawTokenFormatter']
+__all__ = ['NullFormatter', 'RawTokenFormatter', 'TestcaseFormatter']
 
 
 class NullFormatter(Formatter):
@@ -40,7 +40,7 @@ class RawTokenFormatter(Formatter):
 
     The format is ``tokentype<TAB>repr(tokenstring)\n``. The output can later
     be converted to a token stream with the `RawTokenLexer`, described in the
-    `lexer list <lexers.txt>`_.
+    :doc:`lexer list <lexers>`.
 
     Only two options are accepted:
 
@@ -50,7 +50,8 @@ class RawTokenFormatter(Formatter):
     `error_color`
         If set to a color name, highlight error tokens using that color.  If
         set but with no value, defaults to ``'red'``.
-        *New in Pygments 0.11.*
+
+        .. versionadded:: 0.11
 
     """
     name = 'Raw tokens'
@@ -61,9 +62,9 @@ class RawTokenFormatter(Formatter):
 
     def __init__(self, **options):
         Formatter.__init__(self, **options)
-        if self.encoding:
-            raise OptionError('the raw formatter does not support the '
-                              'encoding option')
+        # We ignore self.encoding if it is set, since it gets set for lexer
+        # and formatter if given with -Oencoding on the command line.
+        # The RawTokenFormatter outputs only ASCII. Override here.
         self.encoding = 'ascii'  # let pygments.format() do the right thing
         self.compress = get_choice_opt(options, 'compress',
                                        ['', 'none', 'gz', 'bz2'], '')
@@ -79,7 +80,7 @@ class RawTokenFormatter(Formatter):
 
     def format(self, tokensource, outfile):
         try:
-            outfile.write(b(''))
+            outfile.write(b'')
         except TypeError:
             raise TypeError('The raw tokens formatter needs a binary '
                             'output file')
@@ -113,3 +114,47 @@ class RawTokenFormatter(Formatter):
             for ttype, value in tokensource:
                 write("%s\t%r\n" % (ttype, value))
         flush()
+
+TESTCASE_BEFORE = u'''\
+    def testNeedsName(self):
+        fragment = %r
+        tokens = [
+'''
+TESTCASE_AFTER = u'''\
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+'''
+
+
+class TestcaseFormatter(Formatter):
+    """
+    Format tokens as appropriate for a new testcase.
+
+    .. versionadded:: 2.0
+    """
+    name = 'Testcase'
+    aliases = ['testcase']
+
+    def __init__(self, **options):
+        Formatter.__init__(self, **options)
+        if self.encoding is not None and self.encoding != 'utf-8':
+            raise ValueError("Only None and utf-8 are allowed encodings.")
+
+    def format(self, tokensource, outfile):
+        indentation = ' ' * 12
+        rawbuf = []
+        outbuf = []
+        for ttype, value in tokensource:
+            rawbuf.append(value)
+            outbuf.append('%s(%s, %r),\n' % (indentation, ttype, value))
+
+        before = TESTCASE_BEFORE % (u''.join(rawbuf),)
+        during = u''.join(outbuf)
+        after = TESTCASE_AFTER
+        if self.encoding is None:
+            outfile.write(before + during + after)
+        else:
+            outfile.write(before.encode('utf-8'))
+            outfile.write(during.encode('utf-8'))
+            outfile.write(after.encode('utf-8'))
+        outfile.flush()

packages/wakatime/packages/pygments/formatters/rtf.py

@@ -5,11 +5,12 @@
 
     A formatter that generates RTF files.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 from pygments.formatter import Formatter
+from pygments.util import get_int_opt, _surrogatepair
 
 
 __all__ = ['RtfFormatter']
@@ -19,9 +20,13 @@ class RtfFormatter(Formatter):
     """
     Format tokens as RTF markup. This formatter automatically outputs full RTF
     documents with color information and other useful stuff. Perfect for Copy and
-    Paste into Microsoft® Word® documents.
+    Paste into Microsoft(R) Word(R) documents.
 
-    *New in Pygments 0.6.*
+    Please note that ``encoding`` and ``outencoding`` options are ignored.
+    The RTF format is ASCII natively, but handles unicode characters correctly
+    thanks to escape sequences.
+
+    .. versionadded:: 0.6
 
     Additional options accepted:
 
@@ -32,15 +37,19 @@ class RtfFormatter(Formatter):
     `fontface`
         The used font famliy, for example ``Bitstream Vera Sans``. Defaults to
         some generic font which is supposed to have fixed width.
+
+    `fontsize`
+        Size of the font used. Size is specified in half points. The
+        default is 24 half-points, giving a size 12 font.
+
+        .. versionadded:: 2.0
     """
     name = 'RTF'
     aliases = ['rtf']
     filenames = ['*.rtf']
 
-    unicodeoutput = False
-
     def __init__(self, **options):
-        """
+        r"""
         Additional options accepted:
 
         ``fontface``
@@ -49,48 +58,48 @@ class RtfFormatter(Formatter):
             specification claims that ``\fmodern`` are "Fixed-pitch serif
             and sans serif fonts". Hope every RTF implementation thinks
             the same about modern...
+
         """
         Formatter.__init__(self, **options)
         self.fontface = options.get('fontface') or ''
+        self.fontsize = get_int_opt(options, 'fontsize', 0)
 
     def _escape(self, text):
-        return text.replace('\\', '\\\\') \
-                   .replace('{', '\\{') \
-                   .replace('}', '\\}')
+        return text.replace(u'\\', u'\\\\') \
+                   .replace(u'{', u'\\{') \
+                   .replace(u'}', u'\\}')
 
     def _escape_text(self, text):
         # empty strings, should give a small performance improvment
         if not text:
-            return ''
+            return u''
 
         # escape text
         text = self._escape(text)
-        if self.encoding in ('utf-8', 'utf-16', 'utf-32'):
-            encoding = 'iso-8859-15'
-        else:
-            encoding = self.encoding or 'iso-8859-15'
 
         buf = []
         for c in text:
-            if ord(c) > 128:
-                ansic = c.encode(encoding, 'ignore') or '?'
-                if ord(ansic) > 128:
-                    ansic = '\\\'%x' % ord(ansic)
-                else:
-                    ansic = c
-                buf.append(r'\ud{\u%d%s}' % (ord(c), ansic))
-            else:
+            cn = ord(c)
+            if cn < (2**7):
+                # ASCII character
                 buf.append(str(c))
+            elif (2**7) <= cn < (2**16):
+                # single unicode escape sequence
+                buf.append(u'{\\u%d}' % cn)
+            elif (2**16) <= cn:
+                # RTF limits unicode to 16 bits.
+                # Force surrogate pairs
+                buf.append(u'{\\u%d}{\\u%d}' % _surrogatepair(cn))
 
-        return ''.join(buf).replace('\n', '\\par\n')
+        return u''.join(buf).replace(u'\n', u'\\par\n')
 
     def format_unencoded(self, tokensource, outfile):
         # rtf 1.8 header
-        outfile.write(r'{\rtf1\ansi\deff0'
-                      r'{\fonttbl{\f0\fmodern\fprq1\fcharset0%s;}}'
-                      r'{\colortbl;' % (self.fontface and
-                                        ' ' + self._escape(self.fontface) or
-                                        ''))
+        outfile.write(u'{\\rtf1\\ansi\\uc0\\deff0'
+                      u'{\\fonttbl{\\f0\\fmodern\\fprq1\\fcharset0%s;}}'
+                      u'{\\colortbl;' % (self.fontface and
+                                         u' ' + self._escape(self.fontface) or
+                                         u''))
 
         # convert colors and save them in a mapping to access them later.
         color_mapping = {}
@@ -99,13 +108,15 @@ class RtfFormatter(Formatter):
             for color in style['color'], style['bgcolor'], style['border']:
                 if color and color not in color_mapping:
                     color_mapping[color] = offset
-                    outfile.write(r'\red%d\green%d\blue%d;' % (
+                    outfile.write(u'\\red%d\\green%d\\blue%d;' % (
                         int(color[0:2], 16),
                         int(color[2:4], 16),
                         int(color[4:6], 16)
                     ))
                     offset += 1
-        outfile.write(r'}\f0')
+        outfile.write(u'}\\f0 ')
+        if self.fontsize:
+            outfile.write(u'\\fs%d' % (self.fontsize))
 
         # highlight stream
         for ttype, value in tokensource:
@@ -114,23 +125,23 @@ class RtfFormatter(Formatter):
             style = self.style.style_for_token(ttype)
             buf = []
             if style['bgcolor']:
-                buf.append(r'\cb%d' % color_mapping[style['bgcolor']])
+                buf.append(u'\\cb%d' % color_mapping[style['bgcolor']])
             if style['color']:
-                buf.append(r'\cf%d' % color_mapping[style['color']])
+                buf.append(u'\\cf%d' % color_mapping[style['color']])
             if style['bold']:
-                buf.append(r'\b')
+                buf.append(u'\\b')
             if style['italic']:
-                buf.append(r'\i')
+                buf.append(u'\\i')
             if style['underline']:
-                buf.append(r'\ul')
+                buf.append(u'\\ul')
             if style['border']:
-                buf.append(r'\chbrdr\chcfpat%d' %
+                buf.append(u'\\chbrdr\\chcfpat%d' %
                            color_mapping[style['border']])
-            start = ''.join(buf)
+            start = u''.join(buf)
             if start:
-                outfile.write('{%s ' % start)
+                outfile.write(u'{%s ' % start)
             outfile.write(self._escape_text(value))
             if start:
-                outfile.write('}')
+                outfile.write(u'}')
 
-        outfile.write('}')
+        outfile.write(u'}')

packages/wakatime/packages/pygments/formatters/svg.py

@@ -5,7 +5,7 @@
 
     Formatter for SVG output.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -35,7 +35,7 @@ class SvgFormatter(Formatter):
     By default, this formatter outputs a full SVG document including doctype
     declaration and the ``<svg>`` root element.
 
-    *New in Pygments 0.9.*
+    .. versionadded:: 0.9
 
     Additional options accepted:
 
@@ -78,7 +78,6 @@ class SvgFormatter(Formatter):
     filenames = ['*.svg']
 
     def __init__(self, **options):
-        # XXX outencoding
         Formatter.__init__(self, **options)
         self.nowrap = get_bool_opt(options, 'nowrap', False)
         self.fontfamily = options.get('fontfamily', 'monospace')

packages/wakatime/packages/pygments/formatters/terminal.py

@@ -5,7 +5,7 @@
 
     Formatter for terminal output with ANSI sequences.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -13,7 +13,7 @@ import sys
 
 from pygments.formatter import Formatter
 from pygments.token import Keyword, Name, Comment, String, Error, \
-     Number, Operator, Generic, Token, Whitespace
+    Number, Operator, Generic, Token, Whitespace
 from pygments.console import ansiformat
 from pygments.util import get_choice_opt
 
@@ -49,6 +49,7 @@ TERMINAL_COLORS = {
     Generic.Inserted:   ('darkgreen',  'green'),
     Generic.Heading:    ('**',         '**'),
     Generic.Subheading: ('*purple*',   '*fuchsia*'),
+    Generic.Prompt:     ('**',         '**'),
     Generic.Error:      ('red',        'red'),
 
     Error:              ('_red_',      '_red_'),
@@ -73,6 +74,10 @@ class TerminalFormatter(Formatter):
     `colorscheme`
         A dictionary mapping token types to (lightbg, darkbg) color names or
         ``None`` (default: ``None`` = use builtin colorscheme).
+
+    `linenos`
+        Set to ``True`` to have line numbers on the terminal output as well
+        (default: ``False`` = no line numbers).
     """
     name = 'Terminal'
     aliases = ['terminal', 'console']
@@ -83,6 +88,8 @@ class TerminalFormatter(Formatter):
         self.darkbg = get_choice_opt(options, 'bg',
                                      ['light', 'dark'], 'light') == 'dark'
         self.colorscheme = options.get('colorscheme', None) or TERMINAL_COLORS
+        self.linenos = options.get('linenos', False)
+        self._lineno = 0
 
     def format(self, tokensource, outfile):
         # hack: if the output is a terminal and has an encoding set,
@@ -93,20 +100,37 @@ class TerminalFormatter(Formatter):
             self.encoding = outfile.encoding
         return Formatter.format(self, tokensource, outfile)
 
+    def _write_lineno(self, outfile):
+        self._lineno += 1
+        outfile.write("%s%04d: " % (self._lineno != 1 and '\n' or '', self._lineno))
+
+    def _get_color(self, ttype):
+        # self.colorscheme is a dict containing usually generic types, so we
+        # have to walk the tree of dots.  The base Token type must be a key,
+        # even if it's empty string, as in the default above.
+        colors = self.colorscheme.get(ttype)
+        while colors is None:
+            ttype = ttype.parent
+            colors = self.colorscheme.get(ttype)
+        return colors[self.darkbg]
+
     def format_unencoded(self, tokensource, outfile):
+        if self.linenos:
+            self._write_lineno(outfile)
+
         for ttype, value in tokensource:
-            color = self.colorscheme.get(ttype)
-            while color is None:
-                ttype = ttype[:-1]
-                color = self.colorscheme.get(ttype)
-            if color:
-                color = color[self.darkbg]
-                spl = value.split('\n')
-                for line in spl[:-1]:
-                    if line:
-                        outfile.write(ansiformat(color, line))
-                    outfile.write('\n')
-                if spl[-1]:
-                    outfile.write(ansiformat(color, spl[-1]))
-            else:
-                outfile.write(value)
+            color = self._get_color(ttype)
+
+            for line in value.splitlines(True):
+                if color:
+                    outfile.write(ansiformat(color, line.rstrip('\n')))
+                else:
+                    outfile.write(line.rstrip('\n'))
+                if line.endswith('\n'):
+                    if self.linenos:
+                        self._write_lineno(outfile)
+                    else:
+                        outfile.write('\n')
+
+        if self.linenos:
+            outfile.write("\n")

packages/wakatime/packages/pygments/formatters/terminal256.py

@@ -11,7 +11,7 @@
 
     Formatter version 1.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -27,9 +27,11 @@
 import sys
 
 from pygments.formatter import Formatter
+from pygments.console import codes
+from pygments.style import ansicolors
 
 
-__all__ = ['Terminal256Formatter']
+__all__ = ['Terminal256Formatter', 'TerminalTrueColorFormatter']
 
 
 class EscapeSequence:
@@ -47,9 +49,33 @@ class EscapeSequence:
     def color_string(self):
         attrs = []
         if self.fg is not None:
-            attrs.extend(("38", "5", "%i" % self.fg))
+            if self.fg in ansicolors:
+                esc = codes[self.fg[5:]]
+                if ';01m' in esc:
+                    self.bold = True
+                # extract fg color code.
+                attrs.append(esc[2:4])
+            else:
+                attrs.extend(("38", "5", "%i" % self.fg))
         if self.bg is not None:
-            attrs.extend(("48", "5", "%i" % self.bg))
+            if self.bg in ansicolors:
+                esc = codes[self.bg[5:]]
+                # extract fg color code, add 10 for bg.
+                attrs.append(str(int(esc[2:4])+10))
+            else:
+                attrs.extend(("48", "5", "%i" % self.bg))
+        if self.bold:
+            attrs.append("01")
+        if self.underline:
+            attrs.append("04")
+        return self.escape(attrs)
+
+    def true_color_string(self):
+        attrs = []
+        if self.fg:
+            attrs.extend(("38", "2", str(self.fg[0]), str(self.fg[1]), str(self.fg[2])))
+        if self.bg:
+            attrs.extend(("48", "2", str(self.bg[0]), str(self.bg[1]), str(self.bg[2])))
         if self.bold:
             attrs.append("01")
         if self.underline:
@@ -66,17 +92,23 @@ class EscapeSequence:
             attrs.append("00")
         return self.escape(attrs)
 
+
 class Terminal256Formatter(Formatter):
-    r"""
+    """
     Format tokens with ANSI color sequences, for output in a 256-color
-    terminal or console. Like in `TerminalFormatter` color sequences
+    terminal or console.  Like in `TerminalFormatter` color sequences
     are terminated at newlines, so that paging the output works correctly.
 
     The formatter takes colors from a style defined by the `style` option
     and converts them to nearest ANSI 256-color escape sequences. Bold and
     underline attributes from the style are preserved (and displayed).
 
-    *New in Pygments 0.9.*
+    .. versionadded:: 0.9
+
+    .. versionchanged:: 2.2
+       If the used style defines foreground colors in the form ``#ansi*``, then
+       `Terminal256Formatter` will map these to non extended foreground color.
+       See :ref:`AnsiTerminalStyle` for more information.
 
     Options accepted:
 
@@ -98,28 +130,28 @@ class Terminal256Formatter(Formatter):
         self.usebold = 'nobold' not in options
         self.useunderline = 'nounderline' not in options
 
-        self._build_color_table() # build an RGB-to-256 color conversion table
-        self._setup_styles() # convert selected style's colors to term. colors
+        self._build_color_table()  # build an RGB-to-256 color conversion table
+        self._setup_styles()  # convert selected style's colors to term. colors
 
     def _build_color_table(self):
         # colors 0..15: 16 basic colors
 
-        self.xterm_colors.append((0x00, 0x00, 0x00)) # 0
-        self.xterm_colors.append((0xcd, 0x00, 0x00)) # 1
-        self.xterm_colors.append((0x00, 0xcd, 0x00)) # 2
-        self.xterm_colors.append((0xcd, 0xcd, 0x00)) # 3
-        self.xterm_colors.append((0x00, 0x00, 0xee)) # 4
-        self.xterm_colors.append((0xcd, 0x00, 0xcd)) # 5
-        self.xterm_colors.append((0x00, 0xcd, 0xcd)) # 6
-        self.xterm_colors.append((0xe5, 0xe5, 0xe5)) # 7
-        self.xterm_colors.append((0x7f, 0x7f, 0x7f)) # 8
-        self.xterm_colors.append((0xff, 0x00, 0x00)) # 9
-        self.xterm_colors.append((0x00, 0xff, 0x00)) # 10
-        self.xterm_colors.append((0xff, 0xff, 0x00)) # 11
-        self.xterm_colors.append((0x5c, 0x5c, 0xff)) # 12
-        self.xterm_colors.append((0xff, 0x00, 0xff)) # 13
-        self.xterm_colors.append((0x00, 0xff, 0xff)) # 14
-        self.xterm_colors.append((0xff, 0xff, 0xff)) # 15
+        self.xterm_colors.append((0x00, 0x00, 0x00))  # 0
+        self.xterm_colors.append((0xcd, 0x00, 0x00))  # 1
+        self.xterm_colors.append((0x00, 0xcd, 0x00))  # 2
+        self.xterm_colors.append((0xcd, 0xcd, 0x00))  # 3
+        self.xterm_colors.append((0x00, 0x00, 0xee))  # 4
+        self.xterm_colors.append((0xcd, 0x00, 0xcd))  # 5
+        self.xterm_colors.append((0x00, 0xcd, 0xcd))  # 6
+        self.xterm_colors.append((0xe5, 0xe5, 0xe5))  # 7
+        self.xterm_colors.append((0x7f, 0x7f, 0x7f))  # 8
+        self.xterm_colors.append((0xff, 0x00, 0x00))  # 9
+        self.xterm_colors.append((0x00, 0xff, 0x00))  # 10
+        self.xterm_colors.append((0xff, 0xff, 0x00))  # 11
+        self.xterm_colors.append((0x5c, 0x5c, 0xff))  # 12
+        self.xterm_colors.append((0xff, 0x00, 0xff))  # 13
+        self.xterm_colors.append((0x00, 0xff, 0xff))  # 14
+        self.xterm_colors.append((0xff, 0xff, 0xff))  # 15
 
         # colors 16..232: the 6x6x6 color cube
 
@@ -138,7 +170,7 @@ class Terminal256Formatter(Formatter):
             self.xterm_colors.append((v, v, v))
 
     def _closest_color(self, r, g, b):
-        distance = 257*257*3 # "infinity" (>distance from #000000 to #ffffff)
+        distance = 257*257*3  # "infinity" (>distance from #000000 to #ffffff)
         match = 0
 
         for i in range(0, 254):
@@ -156,6 +188,10 @@ class Terminal256Formatter(Formatter):
 
     def _color_index(self, color):
         index = self.best_match.get(color, None)
+        if color in ansicolors:
+            # strip the `#ansi` part and look up code
+            index = color
+            self.best_match[color] = index
         if index is None:
             try:
                 rgb = int(str(color), 16)
@@ -172,9 +208,14 @@ class Terminal256Formatter(Formatter):
     def _setup_styles(self):
         for ttype, ndef in self.style:
             escape = EscapeSequence()
-            if ndef['color']:
+            # get foreground from ansicolor if set
+            if ndef['ansicolor']:
+                escape.fg = self._color_index(ndef['ansicolor'])
+            elif ndef['color']:
                 escape.fg = self._color_index(ndef['color'])
-            if ndef['bgcolor']:
+            if ndef['bgansicolor']:
+                escape.bg = self._color_index(ndef['bgansicolor'])
+            elif ndef['bgcolor']:
                 escape.bg = self._color_index(ndef['bgcolor'])
             if self.usebold and ndef['bold']:
                 escape.bold = True
@@ -197,7 +238,7 @@ class Terminal256Formatter(Formatter):
             not_found = True
             while ttype and not_found:
                 try:
-                    #outfile.write( "<" + str(ttype) + ">" )
+                    # outfile.write( "<" + str(ttype) + ">" )
                     on, off = self.style_string[str(ttype)]
 
                     # Like TerminalFormatter, add "reset colors" escape sequence
@@ -211,12 +252,58 @@ class Terminal256Formatter(Formatter):
                         outfile.write(on + spl[-1] + off)
 
                     not_found = False
-                    #outfile.write( '#' + str(ttype) + '#' )
+                    # outfile.write( '#' + str(ttype) + '#' )
 
                 except KeyError:
-                    #ottype = ttype
+                    # ottype = ttype
                     ttype = ttype[:-1]
-                    #outfile.write( '!' + str(ottype) + '->' + str(ttype) + '!' )
+                    # outfile.write( '!' + str(ottype) + '->' + str(ttype) + '!' )
 
             if not_found:
                 outfile.write(value)
+
+
+class TerminalTrueColorFormatter(Terminal256Formatter):
+    r"""
+    Format tokens with ANSI color sequences, for output in a true-color
+    terminal or console.  Like in `TerminalFormatter` color sequences
+    are terminated at newlines, so that paging the output works correctly.
+
+    .. versionadded:: 2.1
+
+    Options accepted:
+
+    `style`
+        The style to use, can be a string or a Style subclass (default:
+        ``'default'``).
+    """
+    name = 'TerminalTrueColor'
+    aliases = ['terminal16m', 'console16m', '16m']
+    filenames = []
+
+    def _build_color_table(self):
+        pass
+
+    def _color_tuple(self, color):
+        try:
+            rgb = int(str(color), 16)
+        except ValueError:
+            return None
+        r = (rgb >> 16) & 0xff
+        g = (rgb >> 8) & 0xff
+        b = rgb & 0xff
+        return (r, g, b)
+
+    def _setup_styles(self):
+        for ttype, ndef in self.style:
+            escape = EscapeSequence()
+            if ndef['color']:
+                escape.fg = self._color_tuple(ndef['color'])
+            if ndef['bgcolor']:
+                escape.bg = self._color_tuple(ndef['bgcolor'])
+            if self.usebold and ndef['bold']:
+                escape.bold = True
+            if self.useunderline and ndef['underline']:
+                escape.underline = True
+            self.style_string[str(ttype)] = (escape.true_color_string(),
+                                             escape.reset_string())

packages/wakatime/packages/pygments/lexer.py

@@ -5,27 +5,33 @@
 
     Base lexer classes.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
-import re, itertools
+
+from __future__ import print_function
+
+import re
+import sys
+import time
 
 from pygments.filter import apply_filters, Filter
 from pygments.filters import get_filter_by_name
 from pygments.token import Error, Text, Other, _TokenType
 from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
-     make_analysator
-
+    make_analysator, text_type, add_metaclass, iteritems, Future, guess_decode
+from pygments.regexopt import regex_opt
 
 __all__ = ['Lexer', 'RegexLexer', 'ExtendedRegexLexer', 'DelegatingLexer',
-           'LexerContext', 'include', 'inherit', 'bygroups', 'using', 'this']
+           'LexerContext', 'include', 'inherit', 'bygroups', 'using', 'this',
+           'default', 'words']
 
 
-_encoding_map = [('\xef\xbb\xbf', 'utf-8'),
-                 ('\xff\xfe\0\0', 'utf-32'),
-                 ('\0\0\xfe\xff', 'utf-32be'),
-                 ('\xff\xfe', 'utf-16'),
-                 ('\xfe\xff', 'utf-16be')]
+_encoding_map = [(b'\xef\xbb\xbf', 'utf-8'),
+                 (b'\xff\xfe\0\0', 'utf-32'),
+                 (b'\0\0\xfe\xff', 'utf-32be'),
+                 (b'\xff\xfe', 'utf-16'),
+                 (b'\xfe\xff', 'utf-16be')]
 
 _default_analyse = staticmethod(lambda x: 0.0)
 
@@ -36,12 +42,13 @@ class LexerMeta(type):
     static methods which always return float values.
     """
 
-    def __new__(cls, name, bases, d):
+    def __new__(mcs, name, bases, d):
         if 'analyse_text' in d:
             d['analyse_text'] = make_analysator(d['analyse_text'])
-        return type.__new__(cls, name, bases, d)
+        return type.__new__(mcs, name, bases, d)
 
 
+@add_metaclass(LexerMeta)
 class Lexer(object):
     """
     Lexer for a specific language.
@@ -55,15 +62,19 @@ class Lexer(object):
     ``ensurenl``
         Make sure that the input ends with a newline (default: True).  This
         is required for some lexers that consume input linewise.
-        *New in Pygments 1.3.*
+
+        .. versionadded:: 1.3
+
     ``tabsize``
         If given and greater than 0, expand tabs in the input (default: 0).
     ``encoding``
         If given, must be an encoding name. This encoding will be used to
         convert the input string to Unicode, if it is not already a Unicode
-        string (default: ``'latin1'``).
-        Can also be ``'guess'`` to use a simple UTF-8 / Latin1 detection, or
-        ``'chardet'`` to use the chardet library, if it is installed.
+        string (default: ``'guess'``, which uses a simple UTF-8 / Locale /
+        Latin1 detection.  Can also be ``'chardet'`` to use the chardet
+        library, if it is installed.
+    ``inencoding``
+        Overrides the ``encoding`` if given.
     """
 
     #: Name of the lexer
@@ -84,16 +95,14 @@ class Lexer(object):
     #: Priority, should multiple lexers match and no content is provided
     priority = 0
 
-    __metaclass__ = LexerMeta
-
     def __init__(self, **options):
         self.options = options
         self.stripnl = get_bool_opt(options, 'stripnl', True)
         self.stripall = get_bool_opt(options, 'stripall', False)
         self.ensurenl = get_bool_opt(options, 'ensurenl', True)
         self.tabsize = get_int_opt(options, 'tabsize', 0)
-        self.encoding = options.get('encoding', 'latin1')
-        # self.encoding = options.get('inencoding', None) or self.encoding
+        self.encoding = options.get('encoding', 'guess')
+        self.encoding = options.get('inencoding') or self.encoding
         self.filters = []
         for filter_ in get_list_opt(options, 'filters', ()):
             self.add_filter(filter_)
@@ -136,14 +145,9 @@ class Lexer(object):
         Also preprocess the text, i.e. expand tabs and strip it if
         wanted and applies registered filters.
         """
-        if not isinstance(text, unicode):
+        if not isinstance(text, text_type):
             if self.encoding == 'guess':
-                try:
-                    text = text.decode('utf-8')
-                    if text.startswith(u'\ufeff'):
-                        text = text[len(u'\ufeff'):]
-                except UnicodeDecodeError:
-                    text = text.decode('latin1')
+                text, _ = guess_decode(text)
             elif self.encoding == 'chardet':
                 try:
                     import chardet
@@ -155,17 +159,18 @@ class Lexer(object):
                 decoded = None
                 for bom, encoding in _encoding_map:
                     if text.startswith(bom):
-                        decoded = unicode(text[len(bom):], encoding,
-                                          errors='replace')
+                        decoded = text[len(bom):].decode(encoding, 'replace')
                         break
                 # no BOM found, so use chardet
                 if decoded is None:
-                    enc = chardet.detect(text[:1024]) # Guess using first 1KB
-                    decoded = unicode(text, enc.get('encoding') or 'utf-8',
-                                      errors='replace')
+                    enc = chardet.detect(text[:1024])  # Guess using first 1KB
+                    decoded = text.decode(enc.get('encoding') or 'utf-8',
+                                          'replace')
                 text = decoded
             else:
                 text = text.decode(self.encoding)
+                if text.startswith(u'\ufeff'):
+                    text = text[len(u'\ufeff'):]
         else:
             if text.startswith(u'\ufeff'):
                 text = text[len(u'\ufeff'):]
@@ -183,7 +188,7 @@ class Lexer(object):
             text += '\n'
 
         def streamer():
-            for i, t, v in self.get_tokens_unprocessed(text):
+            for _, t, v in self.get_tokens_unprocessed(text):
                 yield t, v
         stream = streamer()
         if not unfiltered:
@@ -192,7 +197,9 @@ class Lexer(object):
 
     def get_tokens_unprocessed(self, text):
         """
-        Return an iterable of (tokentype, value) pairs.
+        Return an iterable of (index, tokentype, value) pairs where "index"
+        is the starting position of the token within the input text.
+
         In subclasses, implement this method as a generator to
         maximize effectiveness.
         """
@@ -233,12 +240,12 @@ class DelegatingLexer(Lexer):
                              self.root_lexer.get_tokens_unprocessed(buffered))
 
 
-#-------------------------------------------------------------------------------
+# ------------------------------------------------------------------------------
 # RegexLexer and ExtendedRegexLexer
 #
 
 
-class include(str):
+class include(str):  # pylint: disable=invalid-name
     """
     Indicates that a state should include rules from another state.
     """
@@ -252,10 +259,10 @@ class _inherit(object):
     def __repr__(self):
         return 'inherit'
 
-inherit = _inherit()
+inherit = _inherit()  # pylint: disable=invalid-name
 
 
-class combined(tuple):
+class combined(tuple):  # pylint: disable=invalid-name
     """
     Indicates a state combined from multiple states.
     """
@@ -312,8 +319,8 @@ def bygroups(*args):
                 if data is not None:
                     if ctx:
                         ctx.pos = match.start(i + 1)
-                    for item in action(lexer, _PseudoMatch(match.start(i + 1),
-                                       data), ctx):
+                    for item in action(lexer,
+                                       _PseudoMatch(match.start(i + 1), data), ctx):
                         if item:
                             yield item
         if ctx:
@@ -379,20 +386,50 @@ def using(_other, **kwargs):
     return callback
 
 
+class default:
+    """
+    Indicates a state or state action (e.g. #pop) to apply.
+    For example default('#pop') is equivalent to ('', Token, '#pop')
+    Note that state tuples may be used as well.
+
+    .. versionadded:: 2.0
+    """
+    def __init__(self, state):
+        self.state = state
+
+
+class words(Future):
+    """
+    Indicates a list of literal words that is transformed into an optimized
+    regex that matches any of the words.
+
+    .. versionadded:: 2.0
+    """
+    def __init__(self, words, prefix='', suffix=''):
+        self.words = words
+        self.prefix = prefix
+        self.suffix = suffix
+
+    def get(self):
+        return regex_opt(self.words, prefix=self.prefix, suffix=self.suffix)
+
+
 class RegexLexerMeta(LexerMeta):
     """
     Metaclass for RegexLexer, creates the self._tokens attribute from
     self.tokens on the first instantiation.
     """
 
-    def _process_regex(cls, regex, rflags):
+    def _process_regex(cls, regex, rflags, state):
         """Preprocess the regular expression component of a token definition."""
+        if isinstance(regex, Future):
+            regex = regex.get()
         return re.compile(regex, rflags).match
 
     def _process_token(cls, token):
         """Preprocess the token component of a token definition."""
         assert type(token) is _TokenType or callable(token), \
-               'token type must be simple type or callable, not %r' % (token,)
+            'token type must be simple type or callable, not %r' % (token,)
         return token
 
     def _process_new_state(cls, new_state, unprocessed, processed):
@@ -425,7 +462,7 @@ class RegexLexerMeta(LexerMeta):
             for istate in new_state:
                 assert (istate in unprocessed or
                         istate in ('#pop', '#push')), \
-                       'unknown new state ' + istate
+                    'unknown new state ' + istate
             return new_state
         else:
             assert False, 'unknown new state def %r' % new_state
@@ -446,14 +483,20 @@ class RegexLexerMeta(LexerMeta):
                                                  str(tdef)))
                 continue
             if isinstance(tdef, _inherit):
-                # processed already
+                # should be processed already, but may not in the case of:
+                # 1. the state has no counterpart in any parent
+                # 2. the state includes more than one 'inherit'
+                continue
+            if isinstance(tdef, default):
+                new_state = cls._process_new_state(tdef.state, unprocessed, processed)
+                tokens.append((re.compile('').match, None, new_state))
                 continue
 
             assert type(tdef) is tuple, "wrong rule def %r" % tdef
 
             try:
-                rex = cls._process_regex(tdef[0], rflags)
-            except Exception, err:
+                rex = cls._process_regex(tdef[0], rflags, state)
+            except Exception as err:
                 raise ValueError("uncompilable regex %r in state %r of %r: %s" %
                                  (tdef[0], state, cls, err))
 
@@ -472,7 +515,7 @@ class RegexLexerMeta(LexerMeta):
         """Preprocess a dictionary of token definitions."""
         processed = cls._all_tokens[name] = {}
         tokendefs = tokendefs or cls.tokens[name]
-        for state in tokendefs.keys():
+        for state in list(tokendefs):
             cls._process_state(tokendefs, processed, state)
         return processed
 
@@ -490,12 +533,16 @@ class RegexLexerMeta(LexerMeta):
         """
         tokens = {}
         inheritable = {}
-        for c in itertools.chain((cls,), cls.__mro__):
+        for c in cls.__mro__:
             toks = c.__dict__.get('tokens', {})
 
-            for state, items in toks.iteritems():
+            for state, items in iteritems(toks):
                 curitems = tokens.get(state)
                 if curitems is None:
+                    # N.b. because this is assigned by reference, sufficiently
+                    # deep hierarchies are processed incrementally (e.g. for
+                    # A(B), B(C), C(RegexLexer), B will be premodified so X(B)
+                    # will not see any inherits in B).
                     tokens[state] = items
                     try:
                         inherit_ndx = items.index(inherit)
@@ -511,6 +558,8 @@ class RegexLexerMeta(LexerMeta):
                 # Replace the "inherit" value with the items
                 curitems[inherit_ndx:inherit_ndx+1] = items
                 try:
+                    # N.b. this is the index in items (that is, the superclass
+                    # copy), so offset required when storing below.
                     new_inh_ndx = items.index(inherit)
                 except ValueError:
                     pass
@@ -533,13 +582,13 @@ class RegexLexerMeta(LexerMeta):
         return type.__call__(cls, *args, **kwds)
 
 
+@add_metaclass(RegexLexerMeta)
 class RegexLexer(Lexer):
     """
     Base for simple stateful regular expression-based lexers.
     Simplifies the lexing process so that you need only
     provide a list of states and regular expressions.
     """
-    __metaclass__ = RegexLexerMeta
 
     #: Flags for compiling the regular expressions.
     #: Defaults to MULTILINE.
@@ -578,11 +627,12 @@ class RegexLexer(Lexer):
             for rexmatch, action, new_state in statetokens:
                 m = rexmatch(text, pos)
                 if m:
-                    if type(action) is _TokenType:
-                        yield pos, action, m.group()
-                    else:
-                        for item in action(self, m):
-                            yield item
+                    if action is not None:
+                        if type(action) is _TokenType:
+                            yield pos, action, m.group()
+                        else:
+                            for item in action(self, m):
+                                yield item
                     pos = m.end()
                     if new_state is not None:
                         # state transition
@@ -604,6 +654,8 @@ class RegexLexer(Lexer):
                         statetokens = tokendefs[statestack[-1]]
                     break
             else:
+                # We are here only if all state tokens have been considered
+                # and there was not a match on any of them.
                 try:
                     if text[pos] == '\n':
                         # at EOL, reset state to "root"
@@ -626,7 +678,7 @@ class LexerContext(object):
     def __init__(self, text, pos, stack=None, end=None):
         self.text = text
         self.pos = pos
-        self.end = end or len(text) # end=0 not supported ;-)
+        self.end = end or len(text)  # end=0 not supported ;-)
         self.stack = stack or ['root']
 
     def __repr__(self):
@@ -656,15 +708,16 @@ class ExtendedRegexLexer(RegexLexer):
             for rexmatch, action, new_state in statetokens:
                 m = rexmatch(text, ctx.pos, ctx.end)
                 if m:
-                    if type(action) is _TokenType:
-                        yield ctx.pos, action, m.group()
-                        ctx.pos = m.end()
-                    else:
-                        for item in action(self, m, ctx):
-                            yield item
-                        if not new_state:
-                            # altered the state stack?
-                            statetokens = tokendefs[ctx.stack[-1]]
+                    if action is not None:
+                        if type(action) is _TokenType:
+                            yield ctx.pos, action, m.group()
+                            ctx.pos = m.end()
+                        else:
+                            for item in action(self, m, ctx):
+                                yield item
+                            if not new_state:
+                                # altered the state stack?
+                                statetokens = tokendefs[ctx.stack[-1]]
                     # CAUTION: callback must set ctx.pos!
                     if new_state is not None:
                         # state transition
@@ -673,7 +726,7 @@ class ExtendedRegexLexer(RegexLexer):
                                 if state == '#pop':
                                     ctx.stack.pop()
                                 elif state == '#push':
-                                    ctx.stack.append(statestack[-1])
+                                    ctx.stack.append(ctx.stack[-1])
                                 else:
                                     ctx.stack.append(state)
                         elif isinstance(new_state, int):
@@ -718,7 +771,7 @@ def do_insertions(insertions, tokens):
     """
     insertions = iter(insertions)
     try:
-        index, itokens = insertions.next()
+        index, itokens = next(insertions)
     except StopIteration:
         # no insertions
         for item in tokens:
@@ -744,7 +797,7 @@ def do_insertions(insertions, tokens):
                 realpos += len(it_value)
             oldi = index - i
             try:
-                index, itokens = insertions.next()
+                index, itokens = next(insertions)
             except StopIteration:
                 insleft = False
                 break  # not strictly necessary
@@ -759,7 +812,60 @@ def do_insertions(insertions, tokens):
             yield realpos, t, v
             realpos += len(v)
         try:
-            index, itokens = insertions.next()
+            index, itokens = next(insertions)
         except StopIteration:
             insleft = False
             break  # not strictly necessary
+
+
+class ProfilingRegexLexerMeta(RegexLexerMeta):
+    """Metaclass for ProfilingRegexLexer, collects regex timing info."""
+
+    def _process_regex(cls, regex, rflags, state):
+        if isinstance(regex, words):
+            rex = regex_opt(regex.words, prefix=regex.prefix,
+                            suffix=regex.suffix)
+        else:
+            rex = regex
+        compiled = re.compile(rex, rflags)
+
+        def match_func(text, pos, endpos=sys.maxsize):
+            info = cls._prof_data[-1].setdefault((state, rex), [0, 0.0])
+            t0 = time.time()
+            res = compiled.match(text, pos, endpos)
+            t1 = time.time()
+            info[0] += 1
+            info[1] += t1 - t0
+            return res
+        return match_func
+
+
+@add_metaclass(ProfilingRegexLexerMeta)
+class ProfilingRegexLexer(RegexLexer):
+    """Drop-in replacement for RegexLexer that does profiling of its regexes."""
+
+    _prof_data = []
+    _prof_sort_index = 4  # defaults to time per call
+
+    def get_tokens_unprocessed(self, text, stack=('root',)):
+        # this needs to be a stack, since using(this) will produce nested calls
+        self.__class__._prof_data.append({})
+        for tok in RegexLexer.get_tokens_unprocessed(self, text, stack):
+            yield tok
+        rawdata = self.__class__._prof_data.pop()
+        data = sorted(((s, repr(r).strip('u\'').replace('\\\\', '\\')[:65],
+                        n, 1000 * t, 1000 * t / n)
+                       for ((s, r), (n, t)) in rawdata.items()),
+                      key=lambda x: x[self._prof_sort_index],
+                      reverse=True)
+        sum_total = sum(x[3] for x in data)
+
+        print()
+        print('Profiling result for %s lexing %d chars in %.3f ms' %
+              (self.__class__.__name__, len(text), sum_total))
+        print('=' * 110)
+        print('%-20s %-64s ncalls  tottime  percall' % ('state', 'regex'))
+        print('-' * 110)
+        for d in data:
+            print('%-20s %-65s %5d %8.4f %8.4f' % d)
+        print('=' * 110)

packages/wakatime/packages/pygments/lexers/__init__.py

@@ -5,10 +5,11 @@
 
     Pygments lexers.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+import re
 import sys
 import types
 import fnmatch
@@ -17,19 +18,26 @@ from os.path import basename
 from pygments.lexers._mapping import LEXERS
 from pygments.modeline import get_filetype_from_buffer
 from pygments.plugin import find_plugin_lexers
-from pygments.util import ClassNotFound, bytes
+from pygments.util import ClassNotFound, itervalues, guess_decode
 
 
 __all__ = ['get_lexer_by_name', 'get_lexer_for_filename', 'find_lexer_class',
-           'guess_lexer'] + LEXERS.keys()
+           'guess_lexer', 'load_lexer_from_file'] + list(LEXERS)
 
 _lexer_cache = {}
+_pattern_cache = {}
+
+
+def _fn_matches(fn, glob):
+    """Return whether the supplied file name fn matches pattern filename."""
+    if glob not in _pattern_cache:
+        pattern = _pattern_cache[glob] = re.compile(fnmatch.translate(glob))
+        return pattern.match(fn)
+    return _pattern_cache[glob].match(fn)
 
 
 def _load_lexers(module_name):
-    """
-    Load a lexer (and all others in the module too).
-    """
+    """Load a lexer (and all others in the module too)."""
     mod = __import__(module_name, None, None, ['__all__'])
     for lexer_name in mod.__all__:
         cls = getattr(mod, lexer_name)
@@ -37,24 +45,24 @@ def _load_lexers(module_name):
 
 
 def get_all_lexers():
-    """
-    Return a generator of tuples in the form ``(name, aliases,
+    """Return a generator of tuples in the form ``(name, aliases,
     filenames, mimetypes)`` of all know lexers.
     """
-    for item in LEXERS.itervalues():
+    for item in itervalues(LEXERS):
         yield item[1:]
     for lexer in find_plugin_lexers():
         yield lexer.name, lexer.aliases, lexer.filenames, lexer.mimetypes
 
 
 def find_lexer_class(name):
-    """
-    Lookup a lexer class by name. Return None if not found.
+    """Lookup a lexer class by name.
+
+    Return None if not found.
     """
     if name in _lexer_cache:
         return _lexer_cache[name]
     # lookup builtin lexers
-    for module_name, lname, aliases, _, _ in LEXERS.itervalues():
+    for module_name, lname, aliases, _, _ in itervalues(LEXERS):
         if name == lname:
             _load_lexers(module_name)
             return _lexer_cache[name]
@@ -64,45 +72,107 @@ def find_lexer_class(name):
             return cls
 
 
-def get_lexer_by_name(_alias, **options):
-    """
-    Get a lexer by an alias.
+def find_lexer_class_by_name(_alias):
+    """Lookup a lexer class by alias.
+
+    Like `get_lexer_by_name`, but does not instantiate the class.
+
+    .. versionadded:: 2.2
     """
+    if not _alias:
+        raise ClassNotFound('no lexer for alias %r found' % _alias)
     # lookup builtin lexers
-    for module_name, name, aliases, _, _ in LEXERS.itervalues():
-        if _alias in aliases:
+    for module_name, name, aliases, _, _ in itervalues(LEXERS):
+        if _alias.lower() in aliases:
+            if name not in _lexer_cache:
+                _load_lexers(module_name)
+            return _lexer_cache[name]
+    # continue with lexers from setuptools entrypoints
+    for cls in find_plugin_lexers():
+        if _alias.lower() in cls.aliases:
+            return cls
+    raise ClassNotFound('no lexer for alias %r found' % _alias)
+
+
+def get_lexer_by_name(_alias, **options):
+    """Get a lexer by an alias.
+
+    Raises ClassNotFound if not found.
+    """
+    if not _alias:
+        raise ClassNotFound('no lexer for alias %r found' % _alias)
+
+    # lookup builtin lexers
+    for module_name, name, aliases, _, _ in itervalues(LEXERS):
+        if _alias.lower() in aliases:
             if name not in _lexer_cache:
                 _load_lexers(module_name)
             return _lexer_cache[name](**options)
     # continue with lexers from setuptools entrypoints
     for cls in find_plugin_lexers():
-        if _alias in cls.aliases:
+        if _alias.lower() in cls.aliases:
             return cls(**options)
     raise ClassNotFound('no lexer for alias %r found' % _alias)
 
 
-def get_lexer_for_filename(_fn, code=None, **options):
+def load_lexer_from_file(filename, lexername="CustomLexer", **options):
+    """Load a lexer from a file.
+
+    This method expects a file located relative to the current working
+    directory, which contains a Lexer class. By default, it expects the
+    Lexer to be name CustomLexer; you can specify your own class name
+    as the second argument to this function.
+
+    Users should be very careful with the input, because this method
+    is equivalent to running eval on the input file.
+
+    Raises ClassNotFound if there are any problems importing the Lexer.
+
+    .. versionadded:: 2.2
     """
-    Get a lexer for a filename.  If multiple lexers match the filename
-    pattern, use ``analyze_text()`` to figure out which one is more
-    appropriate.
+    try:
+        # This empty dict will contain the namespace for the exec'd file
+        custom_namespace = {}
+        exec(open(filename, 'rb').read(), custom_namespace)
+        # Retrieve the class `lexername` from that namespace
+        if lexername not in custom_namespace:
+            raise ClassNotFound('no valid %s class found in %s' %
+                                (lexername, filename))
+        lexer_class = custom_namespace[lexername]
+        # And finally instantiate it with the options
+        return lexer_class(**options)
+    except IOError as err:
+        raise ClassNotFound('cannot read %s' % filename)
+    except ClassNotFound as err:
+        raise
+    except Exception as err:
+        raise ClassNotFound('error when loading custom lexer: %s' % err)
+
+
+def find_lexer_class_for_filename(_fn, code=None):
+    """Get a lexer for a filename.
+
+    If multiple lexers match the filename pattern, use ``analyse_text()`` to
+    figure out which one is more appropriate.
+
+    Returns None if not found.
     """
     matches = []
     fn = basename(_fn)
-    for modname, name, _, filenames, _ in LEXERS.itervalues():
+    for modname, name, _, filenames, _ in itervalues(LEXERS):
         for filename in filenames:
-            if fnmatch.fnmatch(fn, filename):
+            if _fn_matches(fn, filename):
                 if name not in _lexer_cache:
                     _load_lexers(modname)
                 matches.append((_lexer_cache[name], filename))
     for cls in find_plugin_lexers():
         for filename in cls.filenames:
-            if fnmatch.fnmatch(fn, filename):
+            if _fn_matches(fn, filename):
                 matches.append((cls, filename))
 
     if sys.version_info > (3,) and isinstance(code, bytes):
         # decode it, since all analyse_text functions expect unicode
-        code = code.decode('latin1')
+        code = guess_decode(code)
 
     def get_rating(info):
         cls, filename = info
@@ -113,21 +183,35 @@ def get_lexer_for_filename(_fn, code=None, **options):
         # gets turned into 0.0.  Run scripts/detect_missing_analyse_text.py
         # to find lexers which need it overridden.
         if code:
-            return cls.analyse_text(code) + bonus
-        return cls.priority + bonus
+            return cls.analyse_text(code) + bonus, cls.__name__
+        return cls.priority + bonus, cls.__name__
 
     if matches:
         matches.sort(key=get_rating)
-        #print "Possible lexers, after sort:", matches
-        return matches[-1][0](**options)
-    raise ClassNotFound('no lexer for filename %r found' % _fn)
+        # print "Possible lexers, after sort:", matches
+        return matches[-1][0]
+
+
+def get_lexer_for_filename(_fn, code=None, **options):
+    """Get a lexer for a filename.
+
+    If multiple lexers match the filename pattern, use ``analyse_text()`` to
+    figure out which one is more appropriate.
+
+    Raises ClassNotFound if not found.
+    """
+    res = find_lexer_class_for_filename(_fn, code)
+    if not res:
+        raise ClassNotFound('no lexer for filename %r found' % _fn)
+    return res(**options)
 
 
 def get_lexer_for_mimetype(_mime, **options):
+    """Get a lexer for a mimetype.
+
+    Raises ClassNotFound if not found.
     """
-    Get a lexer for a mimetype.
-    """
-    for modname, name, _, _, mimetypes in LEXERS.itervalues():
+    for modname, name, _, _, mimetypes in itervalues(LEXERS):
         if _mime in mimetypes:
             if name not in _lexer_cache:
                 _load_lexers(modname)
@@ -138,17 +222,16 @@ def get_lexer_for_mimetype(_mime, **options):
     raise ClassNotFound('no lexer for mimetype %r found' % _mime)
 
 
-def _iter_lexerclasses():
-    """
-    Return an iterator over all lexer classes.
-    """
+def _iter_lexerclasses(plugins=True):
+    """Return an iterator over all lexer classes."""
     for key in sorted(LEXERS):
         module_name, name = LEXERS[key][:2]
         if name not in _lexer_cache:
             _load_lexers(module_name)
         yield _lexer_cache[name]
-    for lexer in find_plugin_lexers():
-        yield lexer
+    if plugins:
+        for lexer in find_plugin_lexers():
+            yield lexer
 
 
 def guess_lexer_for_filename(_fn, _text, **options):
@@ -168,16 +251,17 @@ def guess_lexer_for_filename(_fn, _text, **options):
         <pygments.lexers.templates.CssPhpLexer object at 0xb7ba518c>
     """
     fn = basename(_fn)
-    primary = None
+    primary = {}
     matching_lexers = set()
     for lexer in _iter_lexerclasses():
         for filename in lexer.filenames:
-            if fnmatch.fnmatch(fn, filename):
+            if _fn_matches(fn, filename):
                 matching_lexers.add(lexer)
-                primary = lexer
+                primary[lexer] = True
         for filename in lexer.alias_filenames:
-            if fnmatch.fnmatch(fn, filename):
+            if _fn_matches(fn, filename):
                 matching_lexers.add(lexer)
+                primary[lexer] = False
     if not matching_lexers:
         raise ClassNotFound('no lexer for filename %r found' % fn)
     if len(matching_lexers) == 1:
@@ -188,16 +272,21 @@ def guess_lexer_for_filename(_fn, _text, **options):
         if rv == 1.0:
             return lexer(**options)
         result.append((rv, lexer))
-    result.sort()
-    if not result[-1][0] and primary is not None:
-        return primary(**options)
+
+    def type_sort(t):
+        # sort by:
+        # - analyse score
+        # - is primary filename pattern?
+        # - priority
+        # - last resort: class name
+        return (t[0], primary[t[1]], t[1].priority, t[1].__name__)
+    result.sort(key=type_sort)
+
     return result[-1][1](**options)
 
 
 def guess_lexer(_text, **options):
-    """
-    Guess a lexer by strong distinctions in the text (eg, shebang).
-    """
+    """Guess a lexer by strong distinctions in the text (eg, shebang)."""
 
     # try to get a vim modeline first
     ft = get_filetype_from_buffer(_text)
@@ -233,8 +322,8 @@ class _automodule(types.ModuleType):
         raise AttributeError(name)
 
 
-oldmod = sys.modules['pygments.lexers']
-newmod = _automodule('pygments.lexers')
+oldmod = sys.modules[__name__]
+newmod = _automodule(__name__)
 newmod.__dict__.update(oldmod.__dict__)
-sys.modules['pygments.lexers'] = newmod
+sys.modules[__name__] = newmod
 del newmod.newmod, newmod.oldmod, newmod.sys, newmod.types

packages/wakatime/packages/pygments/lexers/_asy_builtins.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
-    pygments.lexers._asybuiltins
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    pygments.lexers._asy_builtins
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     This file contains the asy-function names and asy-variable names of
     Asymptote.
@@ -10,11 +10,11 @@
     TODO: perl/python script in Asymptote SVN similar to asy-list.pl but only
     for function and variable names.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
-ASYFUNCNAME = set([
+ASYFUNCNAME = set((
     'AND',
     'Arc',
     'ArcArrow',
@@ -1038,9 +1038,9 @@ ASYFUNCNAME = set([
     'ztick',
     'ztick3',
     'ztrans'
-])
+))
 
-ASYVARNAME = set([
+ASYVARNAME = set((
     'AliceBlue',
     'Align',
     'Allow',
@@ -1642,4 +1642,4 @@ ASYVARNAME = set([
     'ylabelwidth',
     'zerotickfuzz',
     'zerowinding'
-])
+))

packages/wakatime/packages/pygments/lexers/_cl_builtins.py

@@ -1,15 +1,15 @@
 # -*- coding: utf-8 -*-
 """
-    pygments.lexers._clbuiltins
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    pygments.lexers._cl_builtins
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     ANSI Common Lisp builtins.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
-BUILTIN_FUNCTIONS = [ # 638 functions
+BUILTIN_FUNCTIONS = set((  # 638 functions
     '<', '<=', '=', '>', '>=', '-', '/', '/=', '*', '+', '1-', '1+',
     'abort', 'abs', 'acons', 'acos', 'acosh', 'add-method', 'adjoin',
     'adjustable-array-p', 'adjust-array', 'allocate-instance',
@@ -157,17 +157,17 @@ BUILTIN_FUNCTIONS = [ # 638 functions
     'wild-pathname-p', 'write', 'write-byte', 'write-char', 'write-line',
     'write-sequence', 'write-string', 'write-to-string', 'yes-or-no-p',
     'y-or-n-p', 'zerop',
-]
+))
 
-SPECIAL_FORMS = [
+SPECIAL_FORMS = set((
     'block', 'catch', 'declare', 'eval-when', 'flet', 'function', 'go', 'if',
     'labels', 'lambda', 'let', 'let*', 'load-time-value', 'locally', 'macrolet',
     'multiple-value-call', 'multiple-value-prog1', 'progn', 'progv', 'quote',
     'return-from', 'setq', 'symbol-macrolet', 'tagbody', 'the', 'throw',
     'unwind-protect',
-]
+))
 
-MACROS = [
+MACROS = set((
     'and', 'assert', 'call-method', 'case', 'ccase', 'check-type', 'cond',
     'ctypecase', 'decf', 'declaim', 'defclass', 'defconstant', 'defgeneric',
     'define-compiler-macro', 'define-condition', 'define-method-combination',
@@ -188,19 +188,19 @@ MACROS = [
     'with-input-from-string', 'with-open-file', 'with-open-stream',
     'with-output-to-string', 'with-package-iterator', 'with-simple-restart',
     'with-slots', 'with-standard-io-syntax',
-]
+))
 
-LAMBDA_LIST_KEYWORDS = [
+LAMBDA_LIST_KEYWORDS = set((
     '&allow-other-keys', '&aux', '&body', '&environment', '&key', '&optional',
     '&rest', '&whole',
-]
+))
 
-DECLARATIONS = [
+DECLARATIONS = set((
     'dynamic-extent', 'ignore', 'optimize', 'ftype', 'inline', 'special',
     'ignorable', 'notinline', 'type',
-]
+))
 
-BUILTIN_TYPES = [
+BUILTIN_TYPES = set((
     'atom', 'boolean', 'base-char', 'base-string', 'bignum', 'bit',
     'compiled-function', 'extended-char', 'fixnum', 'keyword', 'nil',
     'signed-byte', 'short-float', 'single-float', 'double-float', 'long-float',
@@ -217,9 +217,9 @@ BUILTIN_TYPES = [
     'simple-type-error', 'simple-warning', 'stream-error', 'storage-condition',
     'style-warning', 'type-error', 'unbound-variable', 'unbound-slot',
     'undefined-function', 'warning',
-]
+))
 
-BUILTIN_CLASSES = [
+BUILTIN_CLASSES = set((
     'array', 'broadcast-stream', 'bit-vector', 'built-in-class', 'character',
     'class', 'complex', 'concatenated-stream', 'cons', 'echo-stream',
     'file-stream', 'float', 'function', 'generic-function', 'hash-table',
@@ -229,4 +229,4 @@ BUILTIN_CLASSES = [
     'standard-generic-function', 'standard-method', 'standard-object',
     'string-stream', 'stream', 'string', 'structure-class', 'structure-object',
     'symbol', 'synonym-stream', 't', 'two-way-stream', 'vector',
-]
+))

packages/wakatime/packages/pygments/lexers/_lua_builtins.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
-    pygments.lexers._luabuiltins
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    pygments.lexers._lua_builtins
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     This file contains the names and modules of lua functions
     It is able to re-generate itself, but for adding new functions you
@@ -9,58 +9,72 @@
 
     Do not edit the MODULES dict by hand.
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
-MODULES = {'basic': ['_G',
+from __future__ import print_function
+
+MODULES = {'basic': ('_G',
            '_VERSION',
            'assert',
            'collectgarbage',
            'dofile',
            'error',
-           'getfenv',
            'getmetatable',
            'ipairs',
            'load',
            'loadfile',
-           'loadstring',
            'next',
            'pairs',
            'pcall',
            'print',
            'rawequal',
            'rawget',
+           'rawlen',
            'rawset',
            'select',
-           'setfenv',
            'setmetatable',
            'tonumber',
            'tostring',
            'type',
-           'unpack',
-           'xpcall'],
- 'coroutine': ['coroutine.create',
+           'xpcall'),
+ 'bit32': ('bit32.arshift',
+           'bit32.band',
+           'bit32.bnot',
+           'bit32.bor',
+           'bit32.btest',
+           'bit32.bxor',
+           'bit32.extract',
+           'bit32.lrotate',
+           'bit32.lshift',
+           'bit32.replace',
+           'bit32.rrotate',
+           'bit32.rshift'),
+ 'coroutine': ('coroutine.create',
+               'coroutine.isyieldable',
                'coroutine.resume',
                'coroutine.running',
                'coroutine.status',
                'coroutine.wrap',
-               'coroutine.yield'],
- 'debug': ['debug.debug',
-           'debug.getfenv',
+               'coroutine.yield'),
+ 'debug': ('debug.debug',
            'debug.gethook',
            'debug.getinfo',
            'debug.getlocal',
            'debug.getmetatable',
            'debug.getregistry',
            'debug.getupvalue',
-           'debug.setfenv',
+           'debug.getuservalue',
            'debug.sethook',
            'debug.setlocal',
            'debug.setmetatable',
            'debug.setupvalue',
-           'debug.traceback'],
- 'io': ['io.close',
+           'debug.setuservalue',
+           'debug.traceback',
+           'debug.upvalueid',
+           'debug.upvaluejoin'),
+ 'io': ('io.close',
         'io.flush',
         'io.input',
         'io.lines',
@@ -68,17 +82,20 @@ MODULES = {'basic': ['_G',
         'io.output',
         'io.popen',
         'io.read',
+        'io.stderr',
+        'io.stdin',
+        'io.stdout',
         'io.tmpfile',
         'io.type',
-        'io.write'],
- 'math': ['math.abs',
+        'io.write'),
+ 'math': ('math.abs',
           'math.acos',
           'math.asin',
-          'math.atan2',
           'math.atan',
+          'math.atan2',
           'math.ceil',
-          'math.cosh',
           'math.cos',
+          'math.cosh',
           'math.deg',
           'math.exp',
           'math.floor',
@@ -86,30 +103,35 @@ MODULES = {'basic': ['_G',
           'math.frexp',
           'math.huge',
           'math.ldexp',
-          'math.log10',
           'math.log',
           'math.max',
+          'math.maxinteger',
           'math.min',
+          'math.mininteger',
           'math.modf',
           'math.pi',
           'math.pow',
           'math.rad',
           'math.random',
           'math.randomseed',
-          'math.sinh',
           'math.sin',
+          'math.sinh',
           'math.sqrt',
+          'math.tan',
           'math.tanh',
-          'math.tan'],
- 'modules': ['module',
-             'require',
+          'math.tointeger',
+          'math.type',
+          'math.ult'),
+ 'modules': ('package.config',
              'package.cpath',
              'package.loaded',
              'package.loadlib',
              'package.path',
              'package.preload',
-             'package.seeall'],
- 'os': ['os.clock',
+             'package.searchers',
+             'package.searchpath',
+             'require'),
+ 'os': ('os.clock',
         'os.date',
         'os.difftime',
         'os.execute',
@@ -119,8 +141,8 @@ MODULES = {'basic': ['_G',
         'os.rename',
         'os.setlocale',
         'os.time',
-        'os.tmpname'],
- 'string': ['string.byte',
+        'os.tmpname'),
+ 'string': ('string.byte',
             'string.char',
             'string.dump',
             'string.find',
@@ -130,19 +152,41 @@ MODULES = {'basic': ['_G',
             'string.len',
             'string.lower',
             'string.match',
+            'string.pack',
+            'string.packsize',
             'string.rep',
             'string.reverse',
             'string.sub',
-            'string.upper'],
- 'table': ['table.concat',
+            'string.unpack',
+            'string.upper'),
+ 'table': ('table.concat',
            'table.insert',
-           'table.maxn',
+           'table.move',
+           'table.pack',
            'table.remove',
-           'table.sort']}
+           'table.sort',
+           'table.unpack'),
+ 'utf8': ('utf8.char',
+          'utf8.charpattern',
+          'utf8.codepoint',
+          'utf8.codes',
+          'utf8.len',
+          'utf8.offset')}
 
-if __name__ == '__main__':
+if __name__ == '__main__':  # pragma: no cover
     import re
-    import urllib.request, urllib.parse, urllib.error
+    import sys
+
+    # urllib ends up wanting to import a module called 'math' -- if
+    # pygments/lexers is in the path, this ends badly.
+    for i in range(len(sys.path)-1, -1, -1):
+        if sys.path[i].endswith('/lexers'):
+            del sys.path[i]
+
+    try:
+        from urllib import urlopen
+    except ImportError:
+        from urllib.request import urlopen
     import pprint
 
     # you can't generally find out what module a function belongs to if you
@@ -188,16 +232,16 @@ if __name__ == '__main__':
 
 
     def get_newest_version():
-        f = urllib.request.urlopen('http://www.lua.org/manual/')
-        r = re.compile(r'^<A HREF="(\d\.\d)/">Lua \1</A>')
+        f = urlopen('http://www.lua.org/manual/')
+        r = re.compile(r'^<A HREF="(\d\.\d)/">(Lua )?\1</A>')
         for line in f:
             m = r.match(line)
             if m is not None:
                 return m.groups()[0]
 
     def get_lua_functions(version):
-        f = urllib.request.urlopen('http://www.lua.org/manual/%s/' % version)
-        r = re.compile(r'^<A HREF="manual.html#pdf-(.+)">\1</A>')
+        f = urlopen('http://www.lua.org/manual/%s/' % version)
+        r = re.compile(r'^<A HREF="manual.html#pdf-(?!lua|LUA)([^:]+)">\1</A>')
         functions = []
         for line in f:
             m = r.match(line)
@@ -215,35 +259,37 @@ if __name__ == '__main__':
             return 'basic'
 
     def regenerate(filename, modules):
-        f = open(filename)
-        try:
-            content = f.read()
-        finally:
-            f.close()
+        with open(filename) as fp:
+            content = fp.read()
 
         header = content[:content.find('MODULES = {')]
         footer = content[content.find("if __name__ == '__main__':"):]
 
 
-        f = open(filename, 'w')
-        f.write(header)
-        f.write('MODULES = %s\n\n' % pprint.pformat(modules))
-        f.write(footer)
-        f.close()
+        with open(filename, 'w') as fp:
+            fp.write(header)
+            fp.write('MODULES = %s\n\n' % pprint.pformat(modules))
+            fp.write(footer)
 
     def run():
         version = get_newest_version()
-        print('> Downloading function index for Lua %s' % version)
-        functions = get_lua_functions(version)
-        print('> %d functions found:' % len(functions))
+        functions = set()
+        for v in ('5.2', version):
+            print('> Downloading function index for Lua %s' % v)
+            f = get_lua_functions(v)
+            print('> %d functions found, %d new:' %
+                  (len(f), len(set(f) - functions)))
+            functions |= set(f)
+
+        functions = sorted(functions)
 
         modules = {}
         for full_function_name in functions:
             print('>> %s' % full_function_name)
             m = get_function_module(full_function_name)
             modules.setdefault(m, []).append(full_function_name)
+        modules = {k: tuple(v) for k, v in modules.iteritems()}
 
         regenerate(__file__, modules)
 
-
     run()

packages/wakatime/packages/pygments/lexers/_mapping.py

@@ -0,0 +1,501 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers._mapping
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer mapping definitions. This file is generated by itself. Everytime
+    you change something on a builtin lexer definition, run this script from
+    the lexers folder to update it.
+
+    Do not alter the LEXERS dictionary by hand.
+
+    :copyright: Copyright 2006-2014, 2016 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from __future__ import print_function
+
+LEXERS = {
+    'ABAPLexer': ('pygments.lexers.business', 'ABAP', ('abap',), ('*.abap', '*.ABAP'), ('text/x-abap',)),
+    'APLLexer': ('pygments.lexers.apl', 'APL', ('apl',), ('*.apl',), ()),
+    'AbnfLexer': ('pygments.lexers.grammar_notation', 'ABNF', ('abnf',), ('*.abnf',), ('text/x-abnf',)),
+    'ActionScript3Lexer': ('pygments.lexers.actionscript', 'ActionScript 3', ('as3', 'actionscript3'), ('*.as',), ('application/x-actionscript3', 'text/x-actionscript3', 'text/actionscript3')),
+    'ActionScriptLexer': ('pygments.lexers.actionscript', 'ActionScript', ('as', 'actionscript'), ('*.as',), ('application/x-actionscript', 'text/x-actionscript', 'text/actionscript')),
+    'AdaLexer': ('pygments.lexers.pascal', 'Ada', ('ada', 'ada95', 'ada2005'), ('*.adb', '*.ads', '*.ada'), ('text/x-ada',)),
+    'AdlLexer': ('pygments.lexers.archetype', 'ADL', ('adl',), ('*.adl', '*.adls', '*.adlf', '*.adlx'), ()),
+    'AgdaLexer': ('pygments.lexers.haskell', 'Agda', ('agda',), ('*.agda',), ('text/x-agda',)),
+    'AheuiLexer': ('pygments.lexers.esoteric', 'Aheui', ('aheui',), ('*.aheui',), ()),
+    'AlloyLexer': ('pygments.lexers.dsls', 'Alloy', ('alloy',), ('*.als',), ('text/x-alloy',)),
+    'AmbientTalkLexer': ('pygments.lexers.ambient', 'AmbientTalk', ('at', 'ambienttalk', 'ambienttalk/2'), ('*.at',), ('text/x-ambienttalk',)),
+    'AmplLexer': ('pygments.lexers.ampl', 'Ampl', ('ampl',), ('*.run',), ()),
+    'Angular2HtmlLexer': ('pygments.lexers.templates', 'HTML + Angular2', ('html+ng2',), ('*.ng2',), ()),
+    'Angular2Lexer': ('pygments.lexers.templates', 'Angular2', ('ng2',), (), ()),
+    'AntlrActionScriptLexer': ('pygments.lexers.parsers', 'ANTLR With ActionScript Target', ('antlr-as', 'antlr-actionscript'), ('*.G', '*.g'), ()),
+    'AntlrCSharpLexer': ('pygments.lexers.parsers', 'ANTLR With C# Target', ('antlr-csharp', 'antlr-c#'), ('*.G', '*.g'), ()),
+    'AntlrCppLexer': ('pygments.lexers.parsers', 'ANTLR With CPP Target', ('antlr-cpp',), ('*.G', '*.g'), ()),
+    'AntlrJavaLexer': ('pygments.lexers.parsers', 'ANTLR With Java Target', ('antlr-java',), ('*.G', '*.g'), ()),
+    'AntlrLexer': ('pygments.lexers.parsers', 'ANTLR', ('antlr',), (), ()),
+    'AntlrObjectiveCLexer': ('pygments.lexers.parsers', 'ANTLR With ObjectiveC Target', ('antlr-objc',), ('*.G', '*.g'), ()),
+    'AntlrPerlLexer': ('pygments.lexers.parsers', 'ANTLR With Perl Target', ('antlr-perl',), ('*.G', '*.g'), ()),
+    'AntlrPythonLexer': ('pygments.lexers.parsers', 'ANTLR With Python Target', ('antlr-python',), ('*.G', '*.g'), ()),
+    'AntlrRubyLexer': ('pygments.lexers.parsers', 'ANTLR With Ruby Target', ('antlr-ruby', 'antlr-rb'), ('*.G', '*.g'), ()),
+    'ApacheConfLexer': ('pygments.lexers.configs', 'ApacheConf', ('apacheconf', 'aconf', 'apache'), ('.htaccess', 'apache.conf', 'apache2.conf'), ('text/x-apacheconf',)),
+    'AppleScriptLexer': ('pygments.lexers.scripting', 'AppleScript', ('applescript',), ('*.applescript',), ()),
+    'ArduinoLexer': ('pygments.lexers.c_like', 'Arduino', ('arduino',), ('*.ino',), ('text/x-arduino',)),
+    'AspectJLexer': ('pygments.lexers.jvm', 'AspectJ', ('aspectj',), ('*.aj',), ('text/x-aspectj',)),
+    'AsymptoteLexer': ('pygments.lexers.graphics', 'Asymptote', ('asy', 'asymptote'), ('*.asy',), ('text/x-asymptote',)),
+    'AutoItLexer': ('pygments.lexers.automation', 'AutoIt', ('autoit',), ('*.au3',), ('text/x-autoit',)),
+    'AutohotkeyLexer': ('pygments.lexers.automation', 'autohotkey', ('ahk', 'autohotkey'), ('*.ahk', '*.ahkl'), ('text/x-autohotkey',)),
+    'AwkLexer': ('pygments.lexers.textedit', 'Awk', ('awk', 'gawk', 'mawk', 'nawk'), ('*.awk',), ('application/x-awk',)),
+    'BBCodeLexer': ('pygments.lexers.markup', 'BBCode', ('bbcode',), (), ('text/x-bbcode',)),
+    'BCLexer': ('pygments.lexers.algebra', 'BC', ('bc',), ('*.bc',), ()),
+    'BSTLexer': ('pygments.lexers.bibtex', 'BST', ('bst', 'bst-pybtex'), ('*.bst',), ()),
+    'BaseMakefileLexer': ('pygments.lexers.make', 'Base Makefile', ('basemake',), (), ()),
+    'BashLexer': ('pygments.lexers.shell', 'Bash', ('bash', 'sh', 'ksh', 'zsh', 'shell'), ('*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass', '*.exheres-0', '*.exlib', '*.zsh', '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'zshrc', '.zshrc', 'PKGBUILD'), ('application/x-sh', 'application/x-shellscript')),
+    'BashSessionLexer': ('pygments.lexers.shell', 'Bash Session', ('console', 'shell-session'), ('*.sh-session', '*.shell-session'), ('application/x-shell-session', 'application/x-sh-session')),
+    'BatchLexer': ('pygments.lexers.shell', 'Batchfile', ('bat', 'batch', 'dosbatch', 'winbatch'), ('*.bat', '*.cmd'), ('application/x-dos-batch',)),
+    'BefungeLexer': ('pygments.lexers.esoteric', 'Befunge', ('befunge',), ('*.befunge',), ('application/x-befunge',)),
+    'BibTeXLexer': ('pygments.lexers.bibtex', 'BibTeX', ('bib', 'bibtex'), ('*.bib',), ('text/x-bibtex',)),
+    'BlitzBasicLexer': ('pygments.lexers.basic', 'BlitzBasic', ('blitzbasic', 'b3d', 'bplus'), ('*.bb', '*.decls'), ('text/x-bb',)),
+    'BlitzMaxLexer': ('pygments.lexers.basic', 'BlitzMax', ('blitzmax', 'bmax'), ('*.bmx',), ('text/x-bmx',)),
+    'BnfLexer': ('pygments.lexers.grammar_notation', 'BNF', ('bnf',), ('*.bnf',), ('text/x-bnf',)),
+    'BooLexer': ('pygments.lexers.dotnet', 'Boo', ('boo',), ('*.boo',), ('text/x-boo',)),
+    'BoogieLexer': ('pygments.lexers.verification', 'Boogie', ('boogie',), ('*.bpl',), ()),
+    'BrainfuckLexer': ('pygments.lexers.esoteric', 'Brainfuck', ('brainfuck', 'bf'), ('*.bf', '*.b'), ('application/x-brainfuck',)),
+    'BroLexer': ('pygments.lexers.dsls', 'Bro', ('bro',), ('*.bro',), ()),
+    'BugsLexer': ('pygments.lexers.modeling', 'BUGS', ('bugs', 'winbugs', 'openbugs'), ('*.bug',), ()),
+    'CAmkESLexer': ('pygments.lexers.esoteric', 'CAmkES', ('camkes', 'idl4'), ('*.camkes', '*.idl4'), ()),
+    'CLexer': ('pygments.lexers.c_cpp', 'C', ('c',), ('*.c', '*.h', '*.idc'), ('text/x-chdr', 'text/x-csrc')),
+    'CMakeLexer': ('pygments.lexers.make', 'CMake', ('cmake',), ('*.cmake', 'CMakeLists.txt'), ('text/x-cmake',)),
+    'CObjdumpLexer': ('pygments.lexers.asm', 'c-objdump', ('c-objdump',), ('*.c-objdump',), ('text/x-c-objdump',)),
+    'CPSALexer': ('pygments.lexers.lisp', 'CPSA', ('cpsa',), ('*.cpsa',), ()),
+    'CSharpAspxLexer': ('pygments.lexers.dotnet', 'aspx-cs', ('aspx-cs',), ('*.aspx', '*.asax', '*.ascx', '*.ashx', '*.asmx', '*.axd'), ()),
+    'CSharpLexer': ('pygments.lexers.dotnet', 'C#', ('csharp', 'c#'), ('*.cs',), ('text/x-csharp',)),
+    'Ca65Lexer': ('pygments.lexers.asm', 'ca65 assembler', ('ca65',), ('*.s',), ()),
+    'CadlLexer': ('pygments.lexers.archetype', 'cADL', ('cadl',), ('*.cadl',), ()),
+    'CapDLLexer': ('pygments.lexers.esoteric', 'CapDL', ('capdl',), ('*.cdl',), ()),
+    'CapnProtoLexer': ('pygments.lexers.capnproto', "Cap'n Proto", ('capnp',), ('*.capnp',), ()),
+    'CbmBasicV2Lexer': ('pygments.lexers.basic', 'CBM BASIC V2', ('cbmbas',), ('*.bas',), ()),
+    'CeylonLexer': ('pygments.lexers.jvm', 'Ceylon', ('ceylon',), ('*.ceylon',), ('text/x-ceylon',)),
+    'Cfengine3Lexer': ('pygments.lexers.configs', 'CFEngine3', ('cfengine3', 'cf3'), ('*.cf',), ()),
+    'ChaiscriptLexer': ('pygments.lexers.scripting', 'ChaiScript', ('chai', 'chaiscript'), ('*.chai',), ('text/x-chaiscript', 'application/x-chaiscript')),
+    'ChapelLexer': ('pygments.lexers.chapel', 'Chapel', ('chapel', 'chpl'), ('*.chpl',), ()),
+    'CheetahHtmlLexer': ('pygments.lexers.templates', 'HTML+Cheetah', ('html+cheetah', 'html+spitfire', 'htmlcheetah'), (), ('text/html+cheetah', 'text/html+spitfire')),
+    'CheetahJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Cheetah', ('js+cheetah', 'javascript+cheetah', 'js+spitfire', 'javascript+spitfire'), (), ('application/x-javascript+cheetah', 'text/x-javascript+cheetah', 'text/javascript+cheetah', 'application/x-javascript+spitfire', 'text/x-javascript+spitfire', 'text/javascript+spitfire')),
+    'CheetahLexer': ('pygments.lexers.templates', 'Cheetah', ('cheetah', 'spitfire'), ('*.tmpl', '*.spt'), ('application/x-cheetah', 'application/x-spitfire')),
+    'CheetahXmlLexer': ('pygments.lexers.templates', 'XML+Cheetah', ('xml+cheetah', 'xml+spitfire'), (), ('application/xml+cheetah', 'application/xml+spitfire')),
+    'CirruLexer': ('pygments.lexers.webmisc', 'Cirru', ('cirru',), ('*.cirru',), ('text/x-cirru',)),
+    'ClayLexer': ('pygments.lexers.c_like', 'Clay', ('clay',), ('*.clay',), ('text/x-clay',)),
+    'CleanLexer': ('pygments.lexers.clean', 'Clean', ('clean',), ('*.icl', '*.dcl'), ()),
+    'ClojureLexer': ('pygments.lexers.jvm', 'Clojure', ('clojure', 'clj'), ('*.clj',), ('text/x-clojure', 'application/x-clojure')),
+    'ClojureScriptLexer': ('pygments.lexers.jvm', 'ClojureScript', ('clojurescript', 'cljs'), ('*.cljs',), ('text/x-clojurescript', 'application/x-clojurescript')),
+    'CobolFreeformatLexer': ('pygments.lexers.business', 'COBOLFree', ('cobolfree',), ('*.cbl', '*.CBL'), ()),
+    'CobolLexer': ('pygments.lexers.business', 'COBOL', ('cobol',), ('*.cob', '*.COB', '*.cpy', '*.CPY'), ('text/x-cobol',)),
+    'CoffeeScriptLexer': ('pygments.lexers.javascript', 'CoffeeScript', ('coffee-script', 'coffeescript', 'coffee'), ('*.coffee',), ('text/coffeescript',)),
+    'ColdfusionCFCLexer': ('pygments.lexers.templates', 'Coldfusion CFC', ('cfc',), ('*.cfc',), ()),
+    'ColdfusionHtmlLexer': ('pygments.lexers.templates', 'Coldfusion HTML', ('cfm',), ('*.cfm', '*.cfml'), ('application/x-coldfusion',)),
+    'ColdfusionLexer': ('pygments.lexers.templates', 'cfstatement', ('cfs',), (), ()),
+    'CommonLispLexer': ('pygments.lexers.lisp', 'Common Lisp', ('common-lisp', 'cl', 'lisp'), ('*.cl', '*.lisp'), ('text/x-common-lisp',)),
+    'ComponentPascalLexer': ('pygments.lexers.oberon', 'Component Pascal', ('componentpascal', 'cp'), ('*.cp', '*.cps'), ('text/x-component-pascal',)),
+    'CoqLexer': ('pygments.lexers.theorem', 'Coq', ('coq',), ('*.v',), ('text/x-coq',)),
+    'CppLexer': ('pygments.lexers.c_cpp', 'C++', ('cpp', 'c++'), ('*.cpp', '*.hpp', '*.c++', '*.h++', '*.cc', '*.hh', '*.cxx', '*.hxx', '*.C', '*.H', '*.cp', '*.CPP'), ('text/x-c++hdr', 'text/x-c++src')),
+    'CppObjdumpLexer': ('pygments.lexers.asm', 'cpp-objdump', ('cpp-objdump', 'c++-objdumb', 'cxx-objdump'), ('*.cpp-objdump', '*.c++-objdump', '*.cxx-objdump'), ('text/x-cpp-objdump',)),
+    'CrmshLexer': ('pygments.lexers.dsls', 'Crmsh', ('crmsh', 'pcmk'), ('*.crmsh', '*.pcmk'), ()),
+    'CrocLexer': ('pygments.lexers.d', 'Croc', ('croc',), ('*.croc',), ('text/x-crocsrc',)),
+    'CryptolLexer': ('pygments.lexers.haskell', 'Cryptol', ('cryptol', 'cry'), ('*.cry',), ('text/x-cryptol',)),
+    'CrystalLexer': ('pygments.lexers.crystal', 'Crystal', ('cr', 'crystal'), ('*.cr',), ('text/x-crystal',)),
+    'CsoundDocumentLexer': ('pygments.lexers.csound', 'Csound Document', ('csound-document', 'csound-csd'), ('*.csd',), ()),
+    'CsoundOrchestraLexer': ('pygments.lexers.csound', 'Csound Orchestra', ('csound', 'csound-orc'), ('*.orc',), ()),
+    'CsoundScoreLexer': ('pygments.lexers.csound', 'Csound Score', ('csound-score', 'csound-sco'), ('*.sco',), ()),
+    'CssDjangoLexer': ('pygments.lexers.templates', 'CSS+Django/Jinja', ('css+django', 'css+jinja'), (), ('text/css+django', 'text/css+jinja')),
+    'CssErbLexer': ('pygments.lexers.templates', 'CSS+Ruby', ('css+erb', 'css+ruby'), (), ('text/css+ruby',)),
+    'CssGenshiLexer': ('pygments.lexers.templates', 'CSS+Genshi Text', ('css+genshitext', 'css+genshi'), (), ('text/css+genshi',)),
+    'CssLexer': ('pygments.lexers.css', 'CSS', ('css',), ('*.css',), ('text/css',)),
+    'CssPhpLexer': ('pygments.lexers.templates', 'CSS+PHP', ('css+php',), (), ('text/css+php',)),
+    'CssSmartyLexer': ('pygments.lexers.templates', 'CSS+Smarty', ('css+smarty',), (), ('text/css+smarty',)),
+    'CudaLexer': ('pygments.lexers.c_like', 'CUDA', ('cuda', 'cu'), ('*.cu', '*.cuh'), ('text/x-cuda',)),
+    'CypherLexer': ('pygments.lexers.graph', 'Cypher', ('cypher',), ('*.cyp', '*.cypher'), ()),
+    'CythonLexer': ('pygments.lexers.python', 'Cython', ('cython', 'pyx', 'pyrex'), ('*.pyx', '*.pxd', '*.pxi'), ('text/x-cython', 'application/x-cython')),
+    'DLexer': ('pygments.lexers.d', 'D', ('d',), ('*.d', '*.di'), ('text/x-dsrc',)),
+    'DObjdumpLexer': ('pygments.lexers.asm', 'd-objdump', ('d-objdump',), ('*.d-objdump',), ('text/x-d-objdump',)),
+    'DarcsPatchLexer': ('pygments.lexers.diff', 'Darcs Patch', ('dpatch',), ('*.dpatch', '*.darcspatch'), ()),
+    'DartLexer': ('pygments.lexers.javascript', 'Dart', ('dart',), ('*.dart',), ('text/x-dart',)),
+    'DebianControlLexer': ('pygments.lexers.installers', 'Debian Control file', ('control', 'debcontrol'), ('control',), ()),
+    'DelphiLexer': ('pygments.lexers.pascal', 'Delphi', ('delphi', 'pas', 'pascal', 'objectpascal'), ('*.pas', '*.dpr'), ('text/x-pascal',)),
+    'DgLexer': ('pygments.lexers.python', 'dg', ('dg',), ('*.dg',), ('text/x-dg',)),
+    'DiffLexer': ('pygments.lexers.diff', 'Diff', ('diff', 'udiff'), ('*.diff', '*.patch'), ('text/x-diff', 'text/x-patch')),
+    'DjangoLexer': ('pygments.lexers.templates', 'Django/Jinja', ('django', 'jinja'), (), ('application/x-django-templating', 'application/x-jinja')),
+    'DockerLexer': ('pygments.lexers.configs', 'Docker', ('docker', 'dockerfile'), ('Dockerfile', '*.docker'), ('text/x-dockerfile-config',)),
+    'DtdLexer': ('pygments.lexers.html', 'DTD', ('dtd',), ('*.dtd',), ('application/xml-dtd',)),
+    'DuelLexer': ('pygments.lexers.webmisc', 'Duel', ('duel', 'jbst', 'jsonml+bst'), ('*.duel', '*.jbst'), ('text/x-duel', 'text/x-jbst')),
+    'DylanConsoleLexer': ('pygments.lexers.dylan', 'Dylan session', ('dylan-console', 'dylan-repl'), ('*.dylan-console',), ('text/x-dylan-console',)),
+    'DylanLexer': ('pygments.lexers.dylan', 'Dylan', ('dylan',), ('*.dylan', '*.dyl', '*.intr'), ('text/x-dylan',)),
+    'DylanLidLexer': ('pygments.lexers.dylan', 'DylanLID', ('dylan-lid', 'lid'), ('*.lid', '*.hdp'), ('text/x-dylan-lid',)),
+    'ECLLexer': ('pygments.lexers.ecl', 'ECL', ('ecl',), ('*.ecl',), ('application/x-ecl',)),
+    'ECLexer': ('pygments.lexers.c_like', 'eC', ('ec',), ('*.ec', '*.eh'), ('text/x-echdr', 'text/x-ecsrc')),
+    'EarlGreyLexer': ('pygments.lexers.javascript', 'Earl Grey', ('earl-grey', 'earlgrey', 'eg'), ('*.eg',), ('text/x-earl-grey',)),
+    'EasytrieveLexer': ('pygments.lexers.scripting', 'Easytrieve', ('easytrieve',), ('*.ezt', '*.mac'), ('text/x-easytrieve',)),
+    'EbnfLexer': ('pygments.lexers.parsers', 'EBNF', ('ebnf',), ('*.ebnf',), ('text/x-ebnf',)),
+    'EiffelLexer': ('pygments.lexers.eiffel', 'Eiffel', ('eiffel',), ('*.e',), ('text/x-eiffel',)),
+    'ElixirConsoleLexer': ('pygments.lexers.erlang', 'Elixir iex session', ('iex',), (), ('text/x-elixir-shellsession',)),
+    'ElixirLexer': ('pygments.lexers.erlang', 'Elixir', ('elixir', 'ex', 'exs'), ('*.ex', '*.exs'), ('text/x-elixir',)),
+    'ElmLexer': ('pygments.lexers.elm', 'Elm', ('elm',), ('*.elm',), ('text/x-elm',)),
+    'EmacsLispLexer': ('pygments.lexers.lisp', 'EmacsLisp', ('emacs', 'elisp', 'emacs-lisp'), ('*.el',), ('text/x-elisp', 'application/x-elisp')),
+    'ErbLexer': ('pygments.lexers.templates', 'ERB', ('erb',), (), ('application/x-ruby-templating',)),
+    'ErlangLexer': ('pygments.lexers.erlang', 'Erlang', ('erlang',), ('*.erl', '*.hrl', '*.es', '*.escript'), ('text/x-erlang',)),
+    'ErlangShellLexer': ('pygments.lexers.erlang', 'Erlang erl session', ('erl',), ('*.erl-sh',), ('text/x-erl-shellsession',)),
+    'EvoqueHtmlLexer': ('pygments.lexers.templates', 'HTML+Evoque', ('html+evoque',), ('*.html',), ('text/html+evoque',)),
+    'EvoqueLexer': ('pygments.lexers.templates', 'Evoque', ('evoque',), ('*.evoque',), ('application/x-evoque',)),
+    'EvoqueXmlLexer': ('pygments.lexers.templates', 'XML+Evoque', ('xml+evoque',), ('*.xml',), ('application/xml+evoque',)),
+    'EzhilLexer': ('pygments.lexers.ezhil', 'Ezhil', ('ezhil',), ('*.n',), ('text/x-ezhil',)),
+    'FSharpLexer': ('pygments.lexers.dotnet', 'FSharp', ('fsharp',), ('*.fs', '*.fsi'), ('text/x-fsharp',)),
+    'FactorLexer': ('pygments.lexers.factor', 'Factor', ('factor',), ('*.factor',), ('text/x-factor',)),
+    'FancyLexer': ('pygments.lexers.ruby', 'Fancy', ('fancy', 'fy'), ('*.fy', '*.fancypack'), ('text/x-fancysrc',)),
+    'FantomLexer': ('pygments.lexers.fantom', 'Fantom', ('fan',), ('*.fan',), ('application/x-fantom',)),
+    'FelixLexer': ('pygments.lexers.felix', 'Felix', ('felix', 'flx'), ('*.flx', '*.flxh'), ('text/x-felix',)),
+    'FishShellLexer': ('pygments.lexers.shell', 'Fish', ('fish', 'fishshell'), ('*.fish', '*.load'), ('application/x-fish',)),
+    'FlatlineLexer': ('pygments.lexers.dsls', 'Flatline', ('flatline',), (), ('text/x-flatline',)),
+    'ForthLexer': ('pygments.lexers.forth', 'Forth', ('forth',), ('*.frt', '*.fs'), ('application/x-forth',)),
+    'FortranFixedLexer': ('pygments.lexers.fortran', 'FortranFixed', ('fortranfixed',), ('*.f', '*.F'), ()),
+    'FortranLexer': ('pygments.lexers.fortran', 'Fortran', ('fortran',), ('*.f03', '*.f90', '*.F03', '*.F90'), ('text/x-fortran',)),
+    'FoxProLexer': ('pygments.lexers.foxpro', 'FoxPro', ('foxpro', 'vfp', 'clipper', 'xbase'), ('*.PRG', '*.prg'), ()),
+    'GAPLexer': ('pygments.lexers.algebra', 'GAP', ('gap',), ('*.g', '*.gd', '*.gi', '*.gap'), ()),
+    'GLShaderLexer': ('pygments.lexers.graphics', 'GLSL', ('glsl',), ('*.vert', '*.frag', '*.geo'), ('text/x-glslsrc',)),
+    'GasLexer': ('pygments.lexers.asm', 'GAS', ('gas', 'asm'), ('*.s', '*.S'), ('text/x-gas',)),
+    'GenshiLexer': ('pygments.lexers.templates', 'Genshi', ('genshi', 'kid', 'xml+genshi', 'xml+kid'), ('*.kid',), ('application/x-genshi', 'application/x-kid')),
+    'GenshiTextLexer': ('pygments.lexers.templates', 'Genshi Text', ('genshitext',), (), ('application/x-genshi-text', 'text/x-genshi')),
+    'GettextLexer': ('pygments.lexers.textfmts', 'Gettext Catalog', ('pot', 'po'), ('*.pot', '*.po'), ('application/x-gettext', 'text/x-gettext', 'text/gettext')),
+    'GherkinLexer': ('pygments.lexers.testing', 'Gherkin', ('cucumber', 'gherkin'), ('*.feature',), ('text/x-gherkin',)),
+    'GnuplotLexer': ('pygments.lexers.graphics', 'Gnuplot', ('gnuplot',), ('*.plot', '*.plt'), ('text/x-gnuplot',)),
+    'GoLexer': ('pygments.lexers.go', 'Go', ('go',), ('*.go',), ('text/x-gosrc',)),
+    'GoloLexer': ('pygments.lexers.jvm', 'Golo', ('golo',), ('*.golo',), ()),
+    'GoodDataCLLexer': ('pygments.lexers.business', 'GoodData-CL', ('gooddata-cl',), ('*.gdc',), ('text/x-gooddata-cl',)),
+    'GosuLexer': ('pygments.lexers.jvm', 'Gosu', ('gosu',), ('*.gs', '*.gsx', '*.gsp', '*.vark'), ('text/x-gosu',)),
+    'GosuTemplateLexer': ('pygments.lexers.jvm', 'Gosu Template', ('gst',), ('*.gst',), ('text/x-gosu-template',)),
+    'GroffLexer': ('pygments.lexers.markup', 'Groff', ('groff', 'nroff', 'man'), ('*.[1234567]', '*.man'), ('application/x-troff', 'text/troff')),
+    'GroovyLexer': ('pygments.lexers.jvm', 'Groovy', ('groovy',), ('*.groovy', '*.gradle'), ('text/x-groovy',)),
+    'HamlLexer': ('pygments.lexers.html', 'Haml', ('haml',), ('*.haml',), ('text/x-haml',)),
+    'HandlebarsHtmlLexer': ('pygments.lexers.templates', 'HTML+Handlebars', ('html+handlebars',), ('*.handlebars', '*.hbs'), ('text/html+handlebars', 'text/x-handlebars-template')),
+    'HandlebarsLexer': ('pygments.lexers.templates', 'Handlebars', ('handlebars',), (), ()),
+    'HaskellLexer': ('pygments.lexers.haskell', 'Haskell', ('haskell', 'hs'), ('*.hs',), ('text/x-haskell',)),
+    'HaxeLexer': ('pygments.lexers.haxe', 'Haxe', ('hx', 'haxe', 'hxsl'), ('*.hx', '*.hxsl'), ('text/haxe', 'text/x-haxe', 'text/x-hx')),
+    'HexdumpLexer': ('pygments.lexers.hexdump', 'Hexdump', ('hexdump',), (), ()),
+    'HsailLexer': ('pygments.lexers.asm', 'HSAIL', ('hsail', 'hsa'), ('*.hsail',), ('text/x-hsail',)),
+    'HtmlDjangoLexer': ('pygments.lexers.templates', 'HTML+Django/Jinja', ('html+django', 'html+jinja', 'htmldjango'), (), ('text/html+django', 'text/html+jinja')),
+    'HtmlGenshiLexer': ('pygments.lexers.templates', 'HTML+Genshi', ('html+genshi', 'html+kid'), (), ('text/html+genshi',)),
+    'HtmlLexer': ('pygments.lexers.html', 'HTML', ('html',), ('*.html', '*.htm', '*.xhtml', '*.xslt'), ('text/html', 'application/xhtml+xml')),
+    'HtmlPhpLexer': ('pygments.lexers.templates', 'HTML+PHP', ('html+php',), ('*.phtml',), ('application/x-php', 'application/x-httpd-php', 'application/x-httpd-php3', 'application/x-httpd-php4', 'application/x-httpd-php5')),
+    'HtmlSmartyLexer': ('pygments.lexers.templates', 'HTML+Smarty', ('html+smarty',), (), ('text/html+smarty',)),
+    'HttpLexer': ('pygments.lexers.textfmts', 'HTTP', ('http',), (), ()),
+    'HxmlLexer': ('pygments.lexers.haxe', 'Hxml', ('haxeml', 'hxml'), ('*.hxml',), ()),
+    'HyLexer': ('pygments.lexers.lisp', 'Hy', ('hylang',), ('*.hy',), ('text/x-hy', 'application/x-hy')),
+    'HybrisLexer': ('pygments.lexers.scripting', 'Hybris', ('hybris', 'hy'), ('*.hy', '*.hyb'), ('text/x-hybris', 'application/x-hybris')),
+    'IDLLexer': ('pygments.lexers.idl', 'IDL', ('idl',), ('*.pro',), ('text/idl',)),
+    'IdrisLexer': ('pygments.lexers.haskell', 'Idris', ('idris', 'idr'), ('*.idr',), ('text/x-idris',)),
+    'IgorLexer': ('pygments.lexers.igor', 'Igor', ('igor', 'igorpro'), ('*.ipf',), ('text/ipf',)),
+    'Inform6Lexer': ('pygments.lexers.int_fiction', 'Inform 6', ('inform6', 'i6'), ('*.inf',), ()),
+    'Inform6TemplateLexer': ('pygments.lexers.int_fiction', 'Inform 6 template', ('i6t',), ('*.i6t',), ()),
+    'Inform7Lexer': ('pygments.lexers.int_fiction', 'Inform 7', ('inform7', 'i7'), ('*.ni', '*.i7x'), ()),
+    'IniLexer': ('pygments.lexers.configs', 'INI', ('ini', 'cfg', 'dosini'), ('*.ini', '*.cfg', '*.inf'), ('text/x-ini', 'text/inf')),
+    'IoLexer': ('pygments.lexers.iolang', 'Io', ('io',), ('*.io',), ('text/x-iosrc',)),
+    'IokeLexer': ('pygments.lexers.jvm', 'Ioke', ('ioke', 'ik'), ('*.ik',), ('text/x-iokesrc',)),
+    'IrcLogsLexer': ('pygments.lexers.textfmts', 'IRC logs', ('irc',), ('*.weechatlog',), ('text/x-irclog',)),
+    'IsabelleLexer': ('pygments.lexers.theorem', 'Isabelle', ('isabelle',), ('*.thy',), ('text/x-isabelle',)),
+    'JLexer': ('pygments.lexers.j', 'J', ('j',), ('*.ijs',), ('text/x-j',)),
+    'JagsLexer': ('pygments.lexers.modeling', 'JAGS', ('jags',), ('*.jag', '*.bug'), ()),
+    'JasminLexer': ('pygments.lexers.jvm', 'Jasmin', ('jasmin', 'jasminxt'), ('*.j',), ()),
+    'JavaLexer': ('pygments.lexers.jvm', 'Java', ('java',), ('*.java',), ('text/x-java',)),
+    'JavascriptDjangoLexer': ('pygments.lexers.templates', 'JavaScript+Django/Jinja', ('js+django', 'javascript+django', 'js+jinja', 'javascript+jinja'), (), ('application/x-javascript+django', 'application/x-javascript+jinja', 'text/x-javascript+django', 'text/x-javascript+jinja', 'text/javascript+django', 'text/javascript+jinja')),
+    'JavascriptErbLexer': ('pygments.lexers.templates', 'JavaScript+Ruby', ('js+erb', 'javascript+erb', 'js+ruby', 'javascript+ruby'), (), ('application/x-javascript+ruby', 'text/x-javascript+ruby', 'text/javascript+ruby')),
+    'JavascriptGenshiLexer': ('pygments.lexers.templates', 'JavaScript+Genshi Text', ('js+genshitext', 'js+genshi', 'javascript+genshitext', 'javascript+genshi'), (), ('application/x-javascript+genshi', 'text/x-javascript+genshi', 'text/javascript+genshi')),
+    'JavascriptLexer': ('pygments.lexers.javascript', 'JavaScript', ('js', 'javascript'), ('*.js', '*.jsm'), ('application/javascript', 'application/x-javascript', 'text/x-javascript', 'text/javascript')),
+    'JavascriptPhpLexer': ('pygments.lexers.templates', 'JavaScript+PHP', ('js+php', 'javascript+php'), (), ('application/x-javascript+php', 'text/x-javascript+php', 'text/javascript+php')),
+    'JavascriptSmartyLexer': ('pygments.lexers.templates', 'JavaScript+Smarty', ('js+smarty', 'javascript+smarty'), (), ('application/x-javascript+smarty', 'text/x-javascript+smarty', 'text/javascript+smarty')),
+    'JclLexer': ('pygments.lexers.scripting', 'JCL', ('jcl',), ('*.jcl',), ('text/x-jcl',)),
+    'JsgfLexer': ('pygments.lexers.grammar_notation', 'JSGF', ('jsgf',), ('*.jsgf',), ('application/jsgf', 'application/x-jsgf', 'text/jsgf')),
+    'JsonBareObjectLexer': ('pygments.lexers.data', 'JSONBareObject', ('json-object',), (), ('application/json-object',)),
+    'JsonLdLexer': ('pygments.lexers.data', 'JSON-LD', ('jsonld', 'json-ld'), ('*.jsonld',), ('application/ld+json',)),
+    'JsonLexer': ('pygments.lexers.data', 'JSON', ('json',), ('*.json',), ('application/json',)),
+    'JspLexer': ('pygments.lexers.templates', 'Java Server Page', ('jsp',), ('*.jsp',), ('application/x-jsp',)),
+    'JuliaConsoleLexer': ('pygments.lexers.julia', 'Julia console', ('jlcon',), (), ()),
+    'JuliaLexer': ('pygments.lexers.julia', 'Julia', ('julia', 'jl'), ('*.jl',), ('text/x-julia', 'application/x-julia')),
+    'JuttleLexer': ('pygments.lexers.javascript', 'Juttle', ('juttle', 'juttle'), ('*.juttle',), ('application/juttle', 'application/x-juttle', 'text/x-juttle', 'text/juttle')),
+    'KalLexer': ('pygments.lexers.javascript', 'Kal', ('kal',), ('*.kal',), ('text/kal', 'application/kal')),
+    'KconfigLexer': ('pygments.lexers.configs', 'Kconfig', ('kconfig', 'menuconfig', 'linux-config', 'kernel-config'), ('Kconfig', '*Config.in*', 'external.in*', 'standard-modules.in'), ('text/x-kconfig',)),
+    'KokaLexer': ('pygments.lexers.haskell', 'Koka', ('koka',), ('*.kk', '*.kki'), ('text/x-koka',)),
+    'KotlinLexer': ('pygments.lexers.jvm', 'Kotlin', ('kotlin',), ('*.kt',), ('text/x-kotlin',)),
+    'LSLLexer': ('pygments.lexers.scripting', 'LSL', ('lsl',), ('*.lsl',), ('text/x-lsl',)),
+    'LassoCssLexer': ('pygments.lexers.templates', 'CSS+Lasso', ('css+lasso',), (), ('text/css+lasso',)),
+    'LassoHtmlLexer': ('pygments.lexers.templates', 'HTML+Lasso', ('html+lasso',), (), ('text/html+lasso', 'application/x-httpd-lasso', 'application/x-httpd-lasso[89]')),
+    'LassoJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Lasso', ('js+lasso', 'javascript+lasso'), (), ('application/x-javascript+lasso', 'text/x-javascript+lasso', 'text/javascript+lasso')),
+    'LassoLexer': ('pygments.lexers.javascript', 'Lasso', ('lasso', 'lassoscript'), ('*.lasso', '*.lasso[89]'), ('text/x-lasso',)),
+    'LassoXmlLexer': ('pygments.lexers.templates', 'XML+Lasso', ('xml+lasso',), (), ('application/xml+lasso',)),
+    'LeanLexer': ('pygments.lexers.theorem', 'Lean', ('lean',), ('*.lean',), ('text/x-lean',)),
+    'LessCssLexer': ('pygments.lexers.css', 'LessCss', ('less',), ('*.less',), ('text/x-less-css',)),
+    'LighttpdConfLexer': ('pygments.lexers.configs', 'Lighttpd configuration file', ('lighty', 'lighttpd'), (), ('text/x-lighttpd-conf',)),
+    'LimboLexer': ('pygments.lexers.inferno', 'Limbo', ('limbo',), ('*.b',), ('text/limbo',)),
+    'LiquidLexer': ('pygments.lexers.templates', 'liquid', ('liquid',), ('*.liquid',), ()),
+    'LiterateAgdaLexer': ('pygments.lexers.haskell', 'Literate Agda', ('lagda', 'literate-agda'), ('*.lagda',), ('text/x-literate-agda',)),
+    'LiterateCryptolLexer': ('pygments.lexers.haskell', 'Literate Cryptol', ('lcry', 'literate-cryptol', 'lcryptol'), ('*.lcry',), ('text/x-literate-cryptol',)),
+    'LiterateHaskellLexer': ('pygments.lexers.haskell', 'Literate Haskell', ('lhs', 'literate-haskell', 'lhaskell'), ('*.lhs',), ('text/x-literate-haskell',)),
+    'LiterateIdrisLexer': ('pygments.lexers.haskell', 'Literate Idris', ('lidr', 'literate-idris', 'lidris'), ('*.lidr',), ('text/x-literate-idris',)),
+    'LiveScriptLexer': ('pygments.lexers.javascript', 'LiveScript', ('live-script', 'livescript'), ('*.ls',), ('text/livescript',)),
+    'LlvmLexer': ('pygments.lexers.asm', 'LLVM', ('llvm',), ('*.ll',), ('text/x-llvm',)),
+    'LogosLexer': ('pygments.lexers.objective', 'Logos', ('logos',), ('*.x', '*.xi', '*.xm', '*.xmi'), ('text/x-logos',)),
+    'LogtalkLexer': ('pygments.lexers.prolog', 'Logtalk', ('logtalk',), ('*.lgt', '*.logtalk'), ('text/x-logtalk',)),
+    'LuaLexer': ('pygments.lexers.scripting', 'Lua', ('lua',), ('*.lua', '*.wlua'), ('text/x-lua', 'application/x-lua')),
+    'MOOCodeLexer': ('pygments.lexers.scripting', 'MOOCode', ('moocode', 'moo'), ('*.moo',), ('text/x-moocode',)),
+    'MSDOSSessionLexer': ('pygments.lexers.shell', 'MSDOS Session', ('doscon',), (), ()),
+    'MakefileLexer': ('pygments.lexers.make', 'Makefile', ('make', 'makefile', 'mf', 'bsdmake'), ('*.mak', '*.mk', 'Makefile', 'makefile', 'Makefile.*', 'GNUmakefile'), ('text/x-makefile',)),
+    'MakoCssLexer': ('pygments.lexers.templates', 'CSS+Mako', ('css+mako',), (), ('text/css+mako',)),
+    'MakoHtmlLexer': ('pygments.lexers.templates', 'HTML+Mako', ('html+mako',), (), ('text/html+mako',)),
+    'MakoJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Mako', ('js+mako', 'javascript+mako'), (), ('application/x-javascript+mako', 'text/x-javascript+mako', 'text/javascript+mako')),
+    'MakoLexer': ('pygments.lexers.templates', 'Mako', ('mako',), ('*.mao',), ('application/x-mako',)),
+    'MakoXmlLexer': ('pygments.lexers.templates', 'XML+Mako', ('xml+mako',), (), ('application/xml+mako',)),
+    'MaqlLexer': ('pygments.lexers.business', 'MAQL', ('maql',), ('*.maql',), ('text/x-gooddata-maql', 'application/x-gooddata-maql')),
+    'MarkdownLexer': ('pygments.lexers.markup', 'markdown', ('md',), ('*.md',), ('text/x-markdown',)),
+    'MaskLexer': ('pygments.lexers.javascript', 'Mask', ('mask',), ('*.mask',), ('text/x-mask',)),
+    'MasonLexer': ('pygments.lexers.templates', 'Mason', ('mason',), ('*.m', '*.mhtml', '*.mc', '*.mi', 'autohandler', 'dhandler'), ('application/x-mason',)),
+    'MathematicaLexer': ('pygments.lexers.algebra', 'Mathematica', ('mathematica', 'mma', 'nb'), ('*.nb', '*.cdf', '*.nbp', '*.ma'), ('application/mathematica', 'application/vnd.wolfram.mathematica', 'application/vnd.wolfram.mathematica.package', 'application/vnd.wolfram.cdf')),
+    'MatlabLexer': ('pygments.lexers.matlab', 'Matlab', ('matlab',), ('*.m',), ('text/matlab',)),
+    'MatlabSessionLexer': ('pygments.lexers.matlab', 'Matlab session', ('matlabsession',), (), ()),
+    'MiniDLexer': ('pygments.lexers.d', 'MiniD', ('minid',), (), ('text/x-minidsrc',)),
+    'ModelicaLexer': ('pygments.lexers.modeling', 'Modelica', ('modelica',), ('*.mo',), ('text/x-modelica',)),
+    'Modula2Lexer': ('pygments.lexers.modula2', 'Modula-2', ('modula2', 'm2'), ('*.def', '*.mod'), ('text/x-modula2',)),
+    'MoinWikiLexer': ('pygments.lexers.markup', 'MoinMoin/Trac Wiki markup', ('trac-wiki', 'moin'), (), ('text/x-trac-wiki',)),
+    'MonkeyLexer': ('pygments.lexers.basic', 'Monkey', ('monkey',), ('*.monkey',), ('text/x-monkey',)),
+    'MonteLexer': ('pygments.lexers.monte', 'Monte', ('monte',), ('*.mt',), ()),
+    'MoonScriptLexer': ('pygments.lexers.scripting', 'MoonScript', ('moon', 'moonscript'), ('*.moon',), ('text/x-moonscript', 'application/x-moonscript')),
+    'MozPreprocCssLexer': ('pygments.lexers.markup', 'CSS+mozpreproc', ('css+mozpreproc',), ('*.css.in',), ()),
+    'MozPreprocHashLexer': ('pygments.lexers.markup', 'mozhashpreproc', ('mozhashpreproc',), (), ()),
+    'MozPreprocJavascriptLexer': ('pygments.lexers.markup', 'Javascript+mozpreproc', ('javascript+mozpreproc',), ('*.js.in',), ()),
+    'MozPreprocPercentLexer': ('pygments.lexers.markup', 'mozpercentpreproc', ('mozpercentpreproc',), (), ()),
+    'MozPreprocXulLexer': ('pygments.lexers.markup', 'XUL+mozpreproc', ('xul+mozpreproc',), ('*.xul.in',), ()),
+    'MqlLexer': ('pygments.lexers.c_like', 'MQL', ('mql', 'mq4', 'mq5', 'mql4', 'mql5'), ('*.mq4', '*.mq5', '*.mqh'), ('text/x-mql',)),
+    'MscgenLexer': ('pygments.lexers.dsls', 'Mscgen', ('mscgen', 'msc'), ('*.msc',), ()),
+    'MuPADLexer': ('pygments.lexers.algebra', 'MuPAD', ('mupad',), ('*.mu',), ()),
+    'MxmlLexer': ('pygments.lexers.actionscript', 'MXML', ('mxml',), ('*.mxml',), ()),
+    'MySqlLexer': ('pygments.lexers.sql', 'MySQL', ('mysql',), (), ('text/x-mysql',)),
+    'MyghtyCssLexer': ('pygments.lexers.templates', 'CSS+Myghty', ('css+myghty',), (), ('text/css+myghty',)),
+    'MyghtyHtmlLexer': ('pygments.lexers.templates', 'HTML+Myghty', ('html+myghty',), (), ('text/html+myghty',)),
+    'MyghtyJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Myghty', ('js+myghty', 'javascript+myghty'), (), ('application/x-javascript+myghty', 'text/x-javascript+myghty', 'text/javascript+mygthy')),
+    'MyghtyLexer': ('pygments.lexers.templates', 'Myghty', ('myghty',), ('*.myt', 'autodelegate'), ('application/x-myghty',)),
+    'MyghtyXmlLexer': ('pygments.lexers.templates', 'XML+Myghty', ('xml+myghty',), (), ('application/xml+myghty',)),
+    'NCLLexer': ('pygments.lexers.ncl', 'NCL', ('ncl',), ('*.ncl',), ('text/ncl',)),
+    'NSISLexer': ('pygments.lexers.installers', 'NSIS', ('nsis', 'nsi', 'nsh'), ('*.nsi', '*.nsh'), ('text/x-nsis',)),
+    'NasmLexer': ('pygments.lexers.asm', 'NASM', ('nasm',), ('*.asm', '*.ASM'), ('text/x-nasm',)),
+    'NasmObjdumpLexer': ('pygments.lexers.asm', 'objdump-nasm', ('objdump-nasm',), ('*.objdump-intel',), ('text/x-nasm-objdump',)),
+    'NemerleLexer': ('pygments.lexers.dotnet', 'Nemerle', ('nemerle',), ('*.n',), ('text/x-nemerle',)),
+    'NesCLexer': ('pygments.lexers.c_like', 'nesC', ('nesc',), ('*.nc',), ('text/x-nescsrc',)),
+    'NewLispLexer': ('pygments.lexers.lisp', 'NewLisp', ('newlisp',), ('*.lsp', '*.nl', '*.kif'), ('text/x-newlisp', 'application/x-newlisp')),
+    'NewspeakLexer': ('pygments.lexers.smalltalk', 'Newspeak', ('newspeak',), ('*.ns2',), ('text/x-newspeak',)),
+    'NginxConfLexer': ('pygments.lexers.configs', 'Nginx configuration file', ('nginx',), ('nginx.conf',), ('text/x-nginx-conf',)),
+    'NimrodLexer': ('pygments.lexers.nimrod', 'Nimrod', ('nim', 'nimrod'), ('*.nim', '*.nimrod'), ('text/x-nim',)),
+    'NitLexer': ('pygments.lexers.nit', 'Nit', ('nit',), ('*.nit',), ()),
+    'NixLexer': ('pygments.lexers.nix', 'Nix', ('nixos', 'nix'), ('*.nix',), ('text/x-nix',)),
+    'NuSMVLexer': ('pygments.lexers.smv', 'NuSMV', ('nusmv',), ('*.smv',), ()),
+    'NumPyLexer': ('pygments.lexers.python', 'NumPy', ('numpy',), (), ()),
+    'ObjdumpLexer': ('pygments.lexers.asm', 'objdump', ('objdump',), ('*.objdump',), ('text/x-objdump',)),
+    'ObjectiveCLexer': ('pygments.lexers.objective', 'Objective-C', ('objective-c', 'objectivec', 'obj-c', 'objc'), ('*.m', '*.h'), ('text/x-objective-c',)),
+    'ObjectiveCppLexer': ('pygments.lexers.objective', 'Objective-C++', ('objective-c++', 'objectivec++', 'obj-c++', 'objc++'), ('*.mm', '*.hh'), ('text/x-objective-c++',)),
+    'ObjectiveJLexer': ('pygments.lexers.javascript', 'Objective-J', ('objective-j', 'objectivej', 'obj-j', 'objj'), ('*.j',), ('text/x-objective-j',)),
+    'OcamlLexer': ('pygments.lexers.ml', 'OCaml', ('ocaml',), ('*.ml', '*.mli', '*.mll', '*.mly'), ('text/x-ocaml',)),
+    'OctaveLexer': ('pygments.lexers.matlab', 'Octave', ('octave',), ('*.m',), ('text/octave',)),
+    'OdinLexer': ('pygments.lexers.archetype', 'ODIN', ('odin',), ('*.odin',), ('text/odin',)),
+    'OocLexer': ('pygments.lexers.ooc', 'Ooc', ('ooc',), ('*.ooc',), ('text/x-ooc',)),
+    'OpaLexer': ('pygments.lexers.ml', 'Opa', ('opa',), ('*.opa',), ('text/x-opa',)),
+    'OpenEdgeLexer': ('pygments.lexers.business', 'OpenEdge ABL', ('openedge', 'abl', 'progress'), ('*.p', '*.cls'), ('text/x-openedge', 'application/x-openedge')),
+    'PacmanConfLexer': ('pygments.lexers.configs', 'PacmanConf', ('pacmanconf',), ('pacman.conf',), ()),
+    'PanLexer': ('pygments.lexers.dsls', 'Pan', ('pan',), ('*.pan',), ()),
+    'ParaSailLexer': ('pygments.lexers.parasail', 'ParaSail', ('parasail',), ('*.psi', '*.psl'), ('text/x-parasail',)),
+    'PawnLexer': ('pygments.lexers.pawn', 'Pawn', ('pawn',), ('*.p', '*.pwn', '*.inc'), ('text/x-pawn',)),
+    'Perl6Lexer': ('pygments.lexers.perl', 'Perl6', ('perl6', 'pl6'), ('*.pl', '*.pm', '*.nqp', '*.p6', '*.6pl', '*.p6l', '*.pl6', '*.6pm', '*.p6m', '*.pm6', '*.t'), ('text/x-perl6', 'application/x-perl6')),
+    'PerlLexer': ('pygments.lexers.perl', 'Perl', ('perl', 'pl'), ('*.pl', '*.pm', '*.t'), ('text/x-perl', 'application/x-perl')),
+    'PhpLexer': ('pygments.lexers.php', 'PHP', ('php', 'php3', 'php4', 'php5'), ('*.php', '*.php[345]', '*.inc'), ('text/x-php',)),
+    'PigLexer': ('pygments.lexers.jvm', 'Pig', ('pig',), ('*.pig',), ('text/x-pig',)),
+    'PikeLexer': ('pygments.lexers.c_like', 'Pike', ('pike',), ('*.pike', '*.pmod'), ('text/x-pike',)),
+    'PkgConfigLexer': ('pygments.lexers.configs', 'PkgConfig', ('pkgconfig',), ('*.pc',), ()),
+    'PlPgsqlLexer': ('pygments.lexers.sql', 'PL/pgSQL', ('plpgsql',), (), ('text/x-plpgsql',)),
+    'PostScriptLexer': ('pygments.lexers.graphics', 'PostScript', ('postscript', 'postscr'), ('*.ps', '*.eps'), ('application/postscript',)),
+    'PostgresConsoleLexer': ('pygments.lexers.sql', 'PostgreSQL console (psql)', ('psql', 'postgresql-console', 'postgres-console'), (), ('text/x-postgresql-psql',)),
+    'PostgresLexer': ('pygments.lexers.sql', 'PostgreSQL SQL dialect', ('postgresql', 'postgres'), (), ('text/x-postgresql',)),
+    'PovrayLexer': ('pygments.lexers.graphics', 'POVRay', ('pov',), ('*.pov', '*.inc'), ('text/x-povray',)),
+    'PowerShellLexer': ('pygments.lexers.shell', 'PowerShell', ('powershell', 'posh', 'ps1', 'psm1'), ('*.ps1', '*.psm1'), ('text/x-powershell',)),
+    'PowerShellSessionLexer': ('pygments.lexers.shell', 'PowerShell Session', ('ps1con',), (), ()),
+    'PraatLexer': ('pygments.lexers.praat', 'Praat', ('praat',), ('*.praat', '*.proc', '*.psc'), ()),
+    'PrologLexer': ('pygments.lexers.prolog', 'Prolog', ('prolog',), ('*.ecl', '*.prolog', '*.pro', '*.pl'), ('text/x-prolog',)),
+    'PropertiesLexer': ('pygments.lexers.configs', 'Properties', ('properties', 'jproperties'), ('*.properties',), ('text/x-java-properties',)),
+    'ProtoBufLexer': ('pygments.lexers.dsls', 'Protocol Buffer', ('protobuf', 'proto'), ('*.proto',), ()),
+    'PugLexer': ('pygments.lexers.html', 'Pug', ('pug', 'jade'), ('*.pug', '*.jade'), ('text/x-pug', 'text/x-jade')),
+    'PuppetLexer': ('pygments.lexers.dsls', 'Puppet', ('puppet',), ('*.pp',), ()),
+    'PyPyLogLexer': ('pygments.lexers.console', 'PyPy Log', ('pypylog', 'pypy'), ('*.pypylog',), ('application/x-pypylog',)),
+    'Python3Lexer': ('pygments.lexers.python', 'Python 3', ('python3', 'py3'), (), ('text/x-python3', 'application/x-python3')),
+    'Python3TracebackLexer': ('pygments.lexers.python', 'Python 3.0 Traceback', ('py3tb',), ('*.py3tb',), ('text/x-python3-traceback',)),
+    'PythonConsoleLexer': ('pygments.lexers.python', 'Python console session', ('pycon',), (), ('text/x-python-doctest',)),
+    'PythonLexer': ('pygments.lexers.python', 'Python', ('python', 'py', 'sage'), ('*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript', '*.tac', '*.sage'), ('text/x-python', 'application/x-python')),
+    'PythonTracebackLexer': ('pygments.lexers.python', 'Python Traceback', ('pytb',), ('*.pytb',), ('text/x-python-traceback',)),
+    'QBasicLexer': ('pygments.lexers.basic', 'QBasic', ('qbasic', 'basic'), ('*.BAS', '*.bas'), ('text/basic',)),
+    'QVToLexer': ('pygments.lexers.qvt', 'QVTO', ('qvto', 'qvt'), ('*.qvto',), ()),
+    'QmlLexer': ('pygments.lexers.webmisc', 'QML', ('qml', 'qbs'), ('*.qml', '*.qbs'), ('application/x-qml', 'application/x-qt.qbs+qml')),
+    'RConsoleLexer': ('pygments.lexers.r', 'RConsole', ('rconsole', 'rout'), ('*.Rout',), ()),
+    'RNCCompactLexer': ('pygments.lexers.rnc', 'Relax-NG Compact', ('rnc', 'rng-compact'), ('*.rnc',), ()),
+    'RPMSpecLexer': ('pygments.lexers.installers', 'RPMSpec', ('spec',), ('*.spec',), ('text/x-rpm-spec',)),
+    'RacketLexer': ('pygments.lexers.lisp', 'Racket', ('racket', 'rkt'), ('*.rkt', '*.rktd', '*.rktl'), ('text/x-racket', 'application/x-racket')),
+    'RagelCLexer': ('pygments.lexers.parsers', 'Ragel in C Host', ('ragel-c',), ('*.rl',), ()),
+    'RagelCppLexer': ('pygments.lexers.parsers', 'Ragel in CPP Host', ('ragel-cpp',), ('*.rl',), ()),
+    'RagelDLexer': ('pygments.lexers.parsers', 'Ragel in D Host', ('ragel-d',), ('*.rl',), ()),
+    'RagelEmbeddedLexer': ('pygments.lexers.parsers', 'Embedded Ragel', ('ragel-em',), ('*.rl',), ()),
+    'RagelJavaLexer': ('pygments.lexers.parsers', 'Ragel in Java Host', ('ragel-java',), ('*.rl',), ()),
+    'RagelLexer': ('pygments.lexers.parsers', 'Ragel', ('ragel',), (), ()),
+    'RagelObjectiveCLexer': ('pygments.lexers.parsers', 'Ragel in Objective C Host', ('ragel-objc',), ('*.rl',), ()),
+    'RagelRubyLexer': ('pygments.lexers.parsers', 'Ragel in Ruby Host', ('ragel-ruby', 'ragel-rb'), ('*.rl',), ()),
+    'RawTokenLexer': ('pygments.lexers.special', 'Raw token data', ('raw',), (), ('application/x-pygments-tokens',)),
+    'RdLexer': ('pygments.lexers.r', 'Rd', ('rd',), ('*.Rd',), ('text/x-r-doc',)),
+    'RebolLexer': ('pygments.lexers.rebol', 'REBOL', ('rebol',), ('*.r', '*.r3', '*.reb'), ('text/x-rebol',)),
+    'RedLexer': ('pygments.lexers.rebol', 'Red', ('red', 'red/system'), ('*.red', '*.reds'), ('text/x-red', 'text/x-red-system')),
+    'RedcodeLexer': ('pygments.lexers.esoteric', 'Redcode', ('redcode',), ('*.cw',), ()),
+    'RegeditLexer': ('pygments.lexers.configs', 'reg', ('registry',), ('*.reg',), ('text/x-windows-registry',)),
+    'ResourceLexer': ('pygments.lexers.resource', 'ResourceBundle', ('resource', 'resourcebundle'), ('*.txt',), ()),
+    'RexxLexer': ('pygments.lexers.scripting', 'Rexx', ('rexx', 'arexx'), ('*.rexx', '*.rex', '*.rx', '*.arexx'), ('text/x-rexx',)),
+    'RhtmlLexer': ('pygments.lexers.templates', 'RHTML', ('rhtml', 'html+erb', 'html+ruby'), ('*.rhtml',), ('text/html+ruby',)),
+    'RoboconfGraphLexer': ('pygments.lexers.roboconf', 'Roboconf Graph', ('roboconf-graph',), ('*.graph',), ()),
+    'RoboconfInstancesLexer': ('pygments.lexers.roboconf', 'Roboconf Instances', ('roboconf-instances',), ('*.instances',), ()),
+    'RobotFrameworkLexer': ('pygments.lexers.robotframework', 'RobotFramework', ('robotframework',), ('*.txt', '*.robot'), ('text/x-robotframework',)),
+    'RqlLexer': ('pygments.lexers.sql', 'RQL', ('rql',), ('*.rql',), ('text/x-rql',)),
+    'RslLexer': ('pygments.lexers.dsls', 'RSL', ('rsl',), ('*.rsl',), ('text/rsl',)),
+    'RstLexer': ('pygments.lexers.markup', 'reStructuredText', ('rst', 'rest', 'restructuredtext'), ('*.rst', '*.rest'), ('text/x-rst', 'text/prs.fallenstein.rst')),
+    'RtsLexer': ('pygments.lexers.trafficscript', 'TrafficScript', ('rts', 'trafficscript'), ('*.rts',), ()),
+    'RubyConsoleLexer': ('pygments.lexers.ruby', 'Ruby irb session', ('rbcon', 'irb'), (), ('text/x-ruby-shellsession',)),
+    'RubyLexer': ('pygments.lexers.ruby', 'Ruby', ('rb', 'ruby', 'duby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby', 'Gemfile'), ('text/x-ruby', 'application/x-ruby')),
+    'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust',), ('*.rs', '*.rs.in'), ('text/rust',)),
+    'SASLexer': ('pygments.lexers.sas', 'SAS', ('sas',), ('*.SAS', '*.sas'), ('text/x-sas', 'text/sas', 'application/x-sas')),
+    'SLexer': ('pygments.lexers.r', 'S', ('splus', 's', 'r'), ('*.S', '*.R', '.Rhistory', '.Rprofile', '.Renviron'), ('text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile')),
+    'SMLLexer': ('pygments.lexers.ml', 'Standard ML', ('sml',), ('*.sml', '*.sig', '*.fun'), ('text/x-standardml', 'application/x-standardml')),
+    'SassLexer': ('pygments.lexers.css', 'Sass', ('sass',), ('*.sass',), ('text/x-sass',)),
+    'ScalaLexer': ('pygments.lexers.jvm', 'Scala', ('scala',), ('*.scala',), ('text/x-scala',)),
+    'ScamlLexer': ('pygments.lexers.html', 'Scaml', ('scaml',), ('*.scaml',), ('text/x-scaml',)),
+    'SchemeLexer': ('pygments.lexers.lisp', 'Scheme', ('scheme', 'scm'), ('*.scm', '*.ss'), ('text/x-scheme', 'application/x-scheme')),
+    'ScilabLexer': ('pygments.lexers.matlab', 'Scilab', ('scilab',), ('*.sci', '*.sce', '*.tst'), ('text/scilab',)),
+    'ScssLexer': ('pygments.lexers.css', 'SCSS', ('scss',), ('*.scss',), ('text/x-scss',)),
+    'ShenLexer': ('pygments.lexers.lisp', 'Shen', ('shen',), ('*.shen',), ('text/x-shen', 'application/x-shen')),
+    'SilverLexer': ('pygments.lexers.verification', 'Silver', ('silver',), ('*.sil', '*.vpr'), ()),
+    'SlimLexer': ('pygments.lexers.webmisc', 'Slim', ('slim',), ('*.slim',), ('text/x-slim',)),
+    'SmaliLexer': ('pygments.lexers.dalvik', 'Smali', ('smali',), ('*.smali',), ('text/smali',)),
+    'SmalltalkLexer': ('pygments.lexers.smalltalk', 'Smalltalk', ('smalltalk', 'squeak', 'st'), ('*.st',), ('text/x-smalltalk',)),
+    'SmartyLexer': ('pygments.lexers.templates', 'Smarty', ('smarty',), ('*.tpl',), ('application/x-smarty',)),
+    'SnobolLexer': ('pygments.lexers.snobol', 'Snobol', ('snobol',), ('*.snobol',), ('text/x-snobol',)),
+    'SnowballLexer': ('pygments.lexers.dsls', 'Snowball', ('snowball',), ('*.sbl',), ()),
+    'SourcePawnLexer': ('pygments.lexers.pawn', 'SourcePawn', ('sp',), ('*.sp',), ('text/x-sourcepawn',)),
+    'SourcesListLexer': ('pygments.lexers.installers', 'Debian Sourcelist', ('sourceslist', 'sources.list', 'debsources'), ('sources.list',), ()),
+    'SparqlLexer': ('pygments.lexers.rdf', 'SPARQL', ('sparql',), ('*.rq', '*.sparql'), ('application/sparql-query',)),
+    'SqlLexer': ('pygments.lexers.sql', 'SQL', ('sql',), ('*.sql',), ('text/x-sql',)),
+    'SqliteConsoleLexer': ('pygments.lexers.sql', 'sqlite3con', ('sqlite3',), ('*.sqlite3-console',), ('text/x-sqlite3-console',)),
+    'SquidConfLexer': ('pygments.lexers.configs', 'SquidConf', ('squidconf', 'squid.conf', 'squid'), ('squid.conf',), ('text/x-squidconf',)),
+    'SspLexer': ('pygments.lexers.templates', 'Scalate Server Page', ('ssp',), ('*.ssp',), ('application/x-ssp',)),
+    'StanLexer': ('pygments.lexers.modeling', 'Stan', ('stan',), ('*.stan',), ()),
+    'StataLexer': ('pygments.lexers.stata', 'Stata', ('stata', 'do'), ('*.do', '*.ado'), ('text/x-stata', 'text/stata', 'application/x-stata')),
+    'SuperColliderLexer': ('pygments.lexers.supercollider', 'SuperCollider', ('sc', 'supercollider'), ('*.sc', '*.scd'), ('application/supercollider', 'text/supercollider')),
+    'SwiftLexer': ('pygments.lexers.objective', 'Swift', ('swift',), ('*.swift',), ('text/x-swift',)),
+    'SwigLexer': ('pygments.lexers.c_like', 'SWIG', ('swig',), ('*.swg', '*.i'), ('text/swig',)),
+    'SystemVerilogLexer': ('pygments.lexers.hdl', 'systemverilog', ('systemverilog', 'sv'), ('*.sv', '*.svh'), ('text/x-systemverilog',)),
+    'TAPLexer': ('pygments.lexers.testing', 'TAP', ('tap',), ('*.tap',), ()),
+    'Tads3Lexer': ('pygments.lexers.int_fiction', 'TADS 3', ('tads3',), ('*.t',), ()),
+    'TasmLexer': ('pygments.lexers.asm', 'TASM', ('tasm',), ('*.asm', '*.ASM', '*.tasm'), ('text/x-tasm',)),
+    'TclLexer': ('pygments.lexers.tcl', 'Tcl', ('tcl',), ('*.tcl', '*.rvt'), ('text/x-tcl', 'text/x-script.tcl', 'application/x-tcl')),
+    'TcshLexer': ('pygments.lexers.shell', 'Tcsh', ('tcsh', 'csh'), ('*.tcsh', '*.csh'), ('application/x-csh',)),
+    'TcshSessionLexer': ('pygments.lexers.shell', 'Tcsh Session', ('tcshcon',), (), ()),
+    'TeaTemplateLexer': ('pygments.lexers.templates', 'Tea', ('tea',), ('*.tea',), ('text/x-tea',)),
+    'TermcapLexer': ('pygments.lexers.configs', 'Termcap', ('termcap',), ('termcap', 'termcap.src'), ()),
+    'TerminfoLexer': ('pygments.lexers.configs', 'Terminfo', ('terminfo',), ('terminfo', 'terminfo.src'), ()),
+    'TerraformLexer': ('pygments.lexers.configs', 'Terraform', ('terraform', 'tf'), ('*.tf',), ('application/x-tf', 'application/x-terraform')),
+    'TexLexer': ('pygments.lexers.markup', 'TeX', ('tex', 'latex'), ('*.tex', '*.aux', '*.toc'), ('text/x-tex', 'text/x-latex')),
+    'TextLexer': ('pygments.lexers.special', 'Text only', ('text',), ('*.txt',), ('text/plain',)),
+    'ThriftLexer': ('pygments.lexers.dsls', 'Thrift', ('thrift',), ('*.thrift',), ('application/x-thrift',)),
+    'TodotxtLexer': ('pygments.lexers.textfmts', 'Todotxt', ('todotxt',), ('todo.txt', '*.todotxt'), ('text/x-todo',)),
+    'TransactSqlLexer': ('pygments.lexers.sql', 'Transact-SQL', ('tsql', 't-sql'), ('*.sql',), ('text/x-tsql',)),
+    'TreetopLexer': ('pygments.lexers.parsers', 'Treetop', ('treetop',), ('*.treetop', '*.tt'), ()),
+    'TurtleLexer': ('pygments.lexers.rdf', 'Turtle', ('turtle',), ('*.ttl',), ('text/turtle', 'application/x-turtle')),
+    'TwigHtmlLexer': ('pygments.lexers.templates', 'HTML+Twig', ('html+twig',), ('*.twig',), ('text/html+twig',)),
+    'TwigLexer': ('pygments.lexers.templates', 'Twig', ('twig',), (), ('application/x-twig',)),
+    'TypeScriptLexer': ('pygments.lexers.javascript', 'TypeScript', ('ts', 'typescript'), ('*.ts', '*.tsx'), ('text/x-typescript',)),
+    'TypoScriptCssDataLexer': ('pygments.lexers.typoscript', 'TypoScriptCssData', ('typoscriptcssdata',), (), ()),
+    'TypoScriptHtmlDataLexer': ('pygments.lexers.typoscript', 'TypoScriptHtmlData', ('typoscripthtmldata',), (), ()),
+    'TypoScriptLexer': ('pygments.lexers.typoscript', 'TypoScript', ('typoscript',), ('*.ts', '*.txt'), ('text/x-typoscript',)),
+    'UrbiscriptLexer': ('pygments.lexers.urbi', 'UrbiScript', ('urbiscript',), ('*.u',), ('application/x-urbiscript',)),
+    'VCLLexer': ('pygments.lexers.varnish', 'VCL', ('vcl',), ('*.vcl',), ('text/x-vclsrc',)),
+    'VCLSnippetLexer': ('pygments.lexers.varnish', 'VCLSnippets', ('vclsnippets', 'vclsnippet'), (), ('text/x-vclsnippet',)),
+    'VCTreeStatusLexer': ('pygments.lexers.console', 'VCTreeStatus', ('vctreestatus',), (), ()),
+    'VGLLexer': ('pygments.lexers.dsls', 'VGL', ('vgl',), ('*.rpf',), ()),
+    'ValaLexer': ('pygments.lexers.c_like', 'Vala', ('vala', 'vapi'), ('*.vala', '*.vapi'), ('text/x-vala',)),
+    'VbNetAspxLexer': ('pygments.lexers.dotnet', 'aspx-vb', ('aspx-vb',), ('*.aspx', '*.asax', '*.ascx', '*.ashx', '*.asmx', '*.axd'), ()),
+    'VbNetLexer': ('pygments.lexers.dotnet', 'VB.net', ('vb.net', 'vbnet'), ('*.vb', '*.bas'), ('text/x-vbnet', 'text/x-vba')),
+    'VelocityHtmlLexer': ('pygments.lexers.templates', 'HTML+Velocity', ('html+velocity',), (), ('text/html+velocity',)),
+    'VelocityLexer': ('pygments.lexers.templates', 'Velocity', ('velocity',), ('*.vm', '*.fhtml'), ()),
+    'VelocityXmlLexer': ('pygments.lexers.templates', 'XML+Velocity', ('xml+velocity',), (), ('application/xml+velocity',)),
+    'VerilogLexer': ('pygments.lexers.hdl', 'verilog', ('verilog', 'v'), ('*.v',), ('text/x-verilog',)),
+    'VhdlLexer': ('pygments.lexers.hdl', 'vhdl', ('vhdl',), ('*.vhdl', '*.vhd'), ('text/x-vhdl',)),
+    'VimLexer': ('pygments.lexers.textedit', 'VimL', ('vim',), ('*.vim', '.vimrc', '.exrc', '.gvimrc', '_vimrc', '_exrc', '_gvimrc', 'vimrc', 'gvimrc'), ('text/x-vim',)),
+    'WDiffLexer': ('pygments.lexers.diff', 'WDiff', ('wdiff',), ('*.wdiff',), ()),
+    'WhileyLexer': ('pygments.lexers.whiley', 'Whiley', ('whiley',), ('*.whiley',), ('text/x-whiley',)),
+    'X10Lexer': ('pygments.lexers.x10', 'X10', ('x10', 'xten'), ('*.x10',), ('text/x-x10',)),
+    'XQueryLexer': ('pygments.lexers.webmisc', 'XQuery', ('xquery', 'xqy', 'xq', 'xql', 'xqm'), ('*.xqy', '*.xquery', '*.xq', '*.xql', '*.xqm'), ('text/xquery', 'application/xquery')),
+    'XmlDjangoLexer': ('pygments.lexers.templates', 'XML+Django/Jinja', ('xml+django', 'xml+jinja'), (), ('application/xml+django', 'application/xml+jinja')),
+    'XmlErbLexer': ('pygments.lexers.templates', 'XML+Ruby', ('xml+erb', 'xml+ruby'), (), ('application/xml+ruby',)),
+    'XmlLexer': ('pygments.lexers.html', 'XML', ('xml',), ('*.xml', '*.xsl', '*.rss', '*.xslt', '*.xsd', '*.wsdl', '*.wsf'), ('text/xml', 'application/xml', 'image/svg+xml', 'application/rss+xml', 'application/atom+xml')),
+    'XmlPhpLexer': ('pygments.lexers.templates', 'XML+PHP', ('xml+php',), (), ('application/xml+php',)),
+    'XmlSmartyLexer': ('pygments.lexers.templates', 'XML+Smarty', ('xml+smarty',), (), ('application/xml+smarty',)),
+    'XorgLexer': ('pygments.lexers.xorg', 'Xorg', ('xorg.conf',), ('xorg.conf',), ()),
+    'XsltLexer': ('pygments.lexers.html', 'XSLT', ('xslt',), ('*.xsl', '*.xslt', '*.xpl'), ('application/xsl+xml', 'application/xslt+xml')),
+    'XtendLexer': ('pygments.lexers.jvm', 'Xtend', ('xtend',), ('*.xtend',), ('text/x-xtend',)),
+    'XtlangLexer': ('pygments.lexers.lisp', 'xtlang', ('extempore',), ('*.xtm',), ()),
+    'YamlJinjaLexer': ('pygments.lexers.templates', 'YAML+Jinja', ('yaml+jinja', 'salt', 'sls'), ('*.sls',), ('text/x-yaml+jinja', 'text/x-sls')),
+    'YamlLexer': ('pygments.lexers.data', 'YAML', ('yaml',), ('*.yaml', '*.yml'), ('text/x-yaml',)),
+    'ZephirLexer': ('pygments.lexers.php', 'Zephir', ('zephir',), ('*.zep',), ()),
+}
+
+if __name__ == '__main__':  # pragma: no cover
+    import sys
+    import os
+
+    # lookup lexers
+    found_lexers = []
+    sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
+    for root, dirs, files in os.walk('.'):
+        for filename in files:
+            if filename.endswith('.py') and not filename.startswith('_'):
+                module_name = 'pygments.lexers%s.%s' % (
+                    root[1:].replace('/', '.'), filename[:-3])
+                print(module_name)
+                module = __import__(module_name, None, None, [''])
+                for lexer_name in module.__all__:
+                    lexer = getattr(module, lexer_name)
+                    found_lexers.append(
+                        '%r: %r' % (lexer_name,
+                                    (module_name,
+                                     lexer.name,
+                                     tuple(lexer.aliases),
+                                     tuple(lexer.filenames),
+                                     tuple(lexer.mimetypes))))
+    # sort them to make the diff minimal
+    found_lexers.sort()
+
+    # extract useful sourcecode from this file
+    with open(__file__) as fp:
+        content = fp.read()
+        # replace crnl to nl for Windows.
+        #
+        # Note that, originally, contributers should keep nl of master
+        # repository, for example by using some kind of automatic
+        # management EOL, like `EolExtension
+        #  <https://www.mercurial-scm.org/wiki/EolExtension>`.
+        content = content.replace("\r\n", "\n")
+    header = content[:content.find('LEXERS = {')]
+    footer = content[content.find("if __name__ == '__main__':"):]
+
+    # write new file
+    with open(__file__, 'w') as fp:
+        fp.write(header)
+        fp.write('LEXERS = {\n    %s,\n}\n\n' % ',\n    '.join(found_lexers))
+        fp.write(footer)
+
+    print ('=== %d lexers processed.' % len(found_lexers))

packages/wakatime/packages/pygments/lexers/_postgres_builtins.py

@@ -0,0 +1,621 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers._postgres_builtins
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Self-updating data files for PostgreSQL lexer.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+
+# Autogenerated: please edit them if you like wasting your time.
+
+KEYWORDS = (
+    'ABORT',
+    'ABSOLUTE',
+    'ACCESS',
+    'ACTION',
+    'ADD',
+    'ADMIN',
+    'AFTER',
+    'AGGREGATE',
+    'ALL',
+    'ALSO',
+    'ALTER',
+    'ALWAYS',
+    'ANALYSE',
+    'ANALYZE',
+    'AND',
+    'ANY',
+    'ARRAY',
+    'AS',
+    'ASC',
+    'ASSERTION',
+    'ASSIGNMENT',
+    'ASYMMETRIC',
+    'AT',
+    'ATTRIBUTE',
+    'AUTHORIZATION',
+    'BACKWARD',
+    'BEFORE',
+    'BEGIN',
+    'BETWEEN',
+    'BIGINT',
+    'BINARY',
+    'BIT',
+    'BOOLEAN',
+    'BOTH',
+    'BY',
+    'CACHE',
+    'CALLED',
+    'CASCADE',
+    'CASCADED',
+    'CASE',
+    'CAST',
+    'CATALOG',
+    'CHAIN',
+    'CHAR',
+    'CHARACTER',
+    'CHARACTERISTICS',
+    'CHECK',
+    'CHECKPOINT',
+    'CLASS',
+    'CLOSE',
+    'CLUSTER',
+    'COALESCE',
+    'COLLATE',
+    'COLLATION',
+    'COLUMN',
+    'COMMENT',
+    'COMMENTS',
+    'COMMIT',
+    'COMMITTED',
+    'CONCURRENTLY',
+    'CONFIGURATION',
+    'CONNECTION',
+    'CONSTRAINT',
+    'CONSTRAINTS',
+    'CONTENT',
+    'CONTINUE',
+    'CONVERSION',
+    'COPY',
+    'COST',
+    'CREATE',
+    'CROSS',
+    'CSV',
+    'CURRENT',
+    'CURRENT_CATALOG',
+    'CURRENT_DATE',
+    'CURRENT_ROLE',
+    'CURRENT_SCHEMA',
+    'CURRENT_TIME',
+    'CURRENT_TIMESTAMP',
+    'CURRENT_USER',
+    'CURSOR',
+    'CYCLE',
+    'DATA',
+    'DATABASE',
+    'DAY',
+    'DEALLOCATE',
+    'DEC',
+    'DECIMAL',
+    'DECLARE',
+    'DEFAULT',
+    'DEFAULTS',
+    'DEFERRABLE',
+    'DEFERRED',
+    'DEFINER',
+    'DELETE',
+    'DELIMITER',
+    'DELIMITERS',
+    'DESC',
+    'DICTIONARY',
+    'DISABLE',
+    'DISCARD',
+    'DISTINCT',
+    'DO',
+    'DOCUMENT',
+    'DOMAIN',
+    'DOUBLE',
+    'DROP',
+    'EACH',
+    'ELSE',
+    'ENABLE',
+    'ENCODING',
+    'ENCRYPTED',
+    'END',
+    'ENUM',
+    'ESCAPE',
+    'EVENT',
+    'EXCEPT',
+    'EXCLUDE',
+    'EXCLUDING',
+    'EXCLUSIVE',
+    'EXECUTE',
+    'EXISTS',
+    'EXPLAIN',
+    'EXTENSION',
+    'EXTERNAL',
+    'EXTRACT',
+    'FALSE',
+    'FAMILY',
+    'FETCH',
+    'FILTER',
+    'FIRST',
+    'FLOAT',
+    'FOLLOWING',
+    'FOR',
+    'FORCE',
+    'FOREIGN',
+    'FORWARD',
+    'FREEZE',
+    'FROM',
+    'FULL',
+    'FUNCTION',
+    'FUNCTIONS',
+    'GLOBAL',
+    'GRANT',
+    'GRANTED',
+    'GREATEST',
+    'GROUP',
+    'HANDLER',
+    'HAVING',
+    'HEADER',
+    'HOLD',
+    'HOUR',
+    'IDENTITY',
+    'IF',
+    'ILIKE',
+    'IMMEDIATE',
+    'IMMUTABLE',
+    'IMPLICIT',
+    'IN',
+    'INCLUDING',
+    'INCREMENT',
+    'INDEX',
+    'INDEXES',
+    'INHERIT',
+    'INHERITS',
+    'INITIALLY',
+    'INLINE',
+    'INNER',
+    'INOUT',
+    'INPUT',
+    'INSENSITIVE',
+    'INSERT',
+    'INSTEAD',
+    'INT',
+    'INTEGER',
+    'INTERSECT',
+    'INTERVAL',
+    'INTO',
+    'INVOKER',
+    'IS',
+    'ISNULL',
+    'ISOLATION',
+    'JOIN',
+    'KEY',
+    'LABEL',
+    'LANGUAGE',
+    'LARGE',
+    'LAST',
+    'LATERAL',
+    'LC_COLLATE',
+    'LC_CTYPE',
+    'LEADING',
+    'LEAKPROOF',
+    'LEAST',
+    'LEFT',
+    'LEVEL',
+    'LIKE',
+    'LIMIT',
+    'LISTEN',
+    'LOAD',
+    'LOCAL',
+    'LOCALTIME',
+    'LOCALTIMESTAMP',
+    'LOCATION',
+    'LOCK',
+    'MAPPING',
+    'MATCH',
+    'MATERIALIZED',
+    'MAXVALUE',
+    'MINUTE',
+    'MINVALUE',
+    'MODE',
+    'MONTH',
+    'MOVE',
+    'NAME',
+    'NAMES',
+    'NATIONAL',
+    'NATURAL',
+    'NCHAR',
+    'NEXT',
+    'NO',
+    'NONE',
+    'NOT',
+    'NOTHING',
+    'NOTIFY',
+    'NOTNULL',
+    'NOWAIT',
+    'NULL',
+    'NULLIF',
+    'NULLS',
+    'NUMERIC',
+    'OBJECT',
+    'OF',
+    'OFF',
+    'OFFSET',
+    'OIDS',
+    'ON',
+    'ONLY',
+    'OPERATOR',
+    'OPTION',
+    'OPTIONS',
+    'OR',
+    'ORDER',
+    'ORDINALITY',
+    'OUT',
+    'OUTER',
+    'OVER',
+    'OVERLAPS',
+    'OVERLAY',
+    'OWNED',
+    'OWNER',
+    'PARSER',
+    'PARTIAL',
+    'PARTITION',
+    'PASSING',
+    'PASSWORD',
+    'PLACING',
+    'PLANS',
+    'POLICY',
+    'POSITION',
+    'PRECEDING',
+    'PRECISION',
+    'PREPARE',
+    'PREPARED',
+    'PRESERVE',
+    'PRIMARY',
+    'PRIOR',
+    'PRIVILEGES',
+    'PROCEDURAL',
+    'PROCEDURE',
+    'PROGRAM',
+    'QUOTE',
+    'RANGE',
+    'READ',
+    'REAL',
+    'REASSIGN',
+    'RECHECK',
+    'RECURSIVE',
+    'REF',
+    'REFERENCES',
+    'REFRESH',
+    'REINDEX',
+    'RELATIVE',
+    'RELEASE',
+    'RENAME',
+    'REPEATABLE',
+    'REPLACE',
+    'REPLICA',
+    'RESET',
+    'RESTART',
+    'RESTRICT',
+    'RETURNING',
+    'RETURNS',
+    'REVOKE',
+    'RIGHT',
+    'ROLE',
+    'ROLLBACK',
+    'ROW',
+    'ROWS',
+    'RULE',
+    'SAVEPOINT',
+    'SCHEMA',
+    'SCROLL',
+    'SEARCH',
+    'SECOND',
+    'SECURITY',
+    'SELECT',
+    'SEQUENCE',
+    'SEQUENCES',
+    'SERIALIZABLE',
+    'SERVER',
+    'SESSION',
+    'SESSION_USER',
+    'SET',
+    'SETOF',
+    'SHARE',
+    'SHOW',
+    'SIMILAR',
+    'SIMPLE',
+    'SMALLINT',
+    'SNAPSHOT',
+    'SOME',
+    'STABLE',
+    'STANDALONE',
+    'START',
+    'STATEMENT',
+    'STATISTICS',
+    'STDIN',
+    'STDOUT',
+    'STORAGE',
+    'STRICT',
+    'STRIP',
+    'SUBSTRING',
+    'SYMMETRIC',
+    'SYSID',
+    'SYSTEM',
+    'TABLE',
+    'TABLES',
+    'TABLESPACE',
+    'TEMP',
+    'TEMPLATE',
+    'TEMPORARY',
+    'TEXT',
+    'THEN',
+    'TIME',
+    'TIMESTAMP',
+    'TO',
+    'TRAILING',
+    'TRANSACTION',
+    'TREAT',
+    'TRIGGER',
+    'TRIM',
+    'TRUE',
+    'TRUNCATE',
+    'TRUSTED',
+    'TYPE',
+    'TYPES',
+    'UNBOUNDED',
+    'UNCOMMITTED',
+    'UNENCRYPTED',
+    'UNION',
+    'UNIQUE',
+    'UNKNOWN',
+    'UNLISTEN',
+    'UNLOGGED',
+    'UNTIL',
+    'UPDATE',
+    'USER',
+    'USING',
+    'VACUUM',
+    'VALID',
+    'VALIDATE',
+    'VALIDATOR',
+    'VALUE',
+    'VALUES',
+    'VARCHAR',
+    'VARIADIC',
+    'VARYING',
+    'VERBOSE',
+    'VERSION',
+    'VIEW',
+    'VIEWS',
+    'VOLATILE',
+    'WHEN',
+    'WHERE',
+    'WHITESPACE',
+    'WINDOW',
+    'WITH',
+    'WITHIN',
+    'WITHOUT',
+    'WORK',
+    'WRAPPER',
+    'WRITE',
+    'XML',
+    'XMLATTRIBUTES',
+    'XMLCONCAT',
+    'XMLELEMENT',
+    'XMLEXISTS',
+    'XMLFOREST',
+    'XMLPARSE',
+    'XMLPI',
+    'XMLROOT',
+    'XMLSERIALIZE',
+    'YEAR',
+    'YES',
+    'ZONE',
+)
+
+DATATYPES = (
+    'bigint',
+    'bigserial',
+    'bit',
+    'bit varying',
+    'bool',
+    'boolean',
+    'box',
+    'bytea',
+    'char',
+    'character',
+    'character varying',
+    'cidr',
+    'circle',
+    'date',
+    'decimal',
+    'double precision',
+    'float4',
+    'float8',
+    'inet',
+    'int',
+    'int2',
+    'int4',
+    'int8',
+    'integer',
+    'interval',
+    'json',
+    'jsonb',
+    'line',
+    'lseg',
+    'macaddr',
+    'money',
+    'numeric',
+    'path',
+    'pg_lsn',
+    'point',
+    'polygon',
+    'real',
+    'serial',
+    'serial2',
+    'serial4',
+    'serial8',
+    'smallint',
+    'smallserial',
+    'text',
+    'time',
+    'timestamp',
+    'timestamptz',
+    'timetz',
+    'tsquery',
+    'tsvector',
+    'txid_snapshot',
+    'uuid',
+    'varbit',
+    'varchar',
+    'with time zone',
+    'without time zone',
+    'xml',
+)
+
+PSEUDO_TYPES = (
+    'any',
+    'anyelement',
+    'anyarray',
+    'anynonarray',
+    'anyenum',
+    'anyrange',
+    'cstring',
+    'internal',
+    'language_handler',
+    'fdw_handler',
+    'record',
+    'trigger',
+    'void',
+    'opaque',
+)
+
+# Remove 'trigger' from types
+PSEUDO_TYPES = tuple(sorted(set(PSEUDO_TYPES) - set(map(str.lower, KEYWORDS))))
+
+PLPGSQL_KEYWORDS = (
+    'ALIAS', 'CONSTANT', 'DIAGNOSTICS', 'ELSIF', 'EXCEPTION', 'EXIT',
+    'FOREACH', 'GET', 'LOOP', 'NOTICE', 'OPEN', 'PERFORM', 'QUERY', 'RAISE',
+    'RETURN', 'REVERSE', 'SQLSTATE', 'WHILE',
+)
+
+
+if __name__ == '__main__':  # pragma: no cover
+    import re
+    try:
+        from urllib import urlopen
+    except ImportError:
+        from urllib.request import urlopen
+
+    from pygments.util import format_lines
+
+    # One man's constant is another man's variable.
+    SOURCE_URL = 'https://github.com/postgres/postgres/raw/master'
+    KEYWORDS_URL = SOURCE_URL + '/doc/src/sgml/keywords.sgml'
+    DATATYPES_URL = SOURCE_URL + '/doc/src/sgml/datatype.sgml'
+
+    def update_myself():
+        data_file = list(urlopen(DATATYPES_URL))
+        datatypes = parse_datatypes(data_file)
+        pseudos = parse_pseudos(data_file)
+
+        keywords = parse_keywords(urlopen(KEYWORDS_URL))
+        update_consts(__file__, 'DATATYPES', datatypes)
+        update_consts(__file__, 'PSEUDO_TYPES', pseudos)
+        update_consts(__file__, 'KEYWORDS', keywords)
+
+    def parse_keywords(f):
+        kw = []
+        for m in re.finditer(
+                r'\s*<entry><token>([^<]+)</token></entry>\s*'
+                r'<entry>([^<]+)</entry>', f.read()):
+            kw.append(m.group(1))
+
+        if not kw:
+            raise ValueError('no keyword found')
+
+        kw.sort()
+        return kw
+
+    def parse_datatypes(f):
+        dt = set()
+        for line in f:
+            if '<sect1' in line:
+                break
+            if '<entry><type>' not in line:
+                continue
+
+            # Parse a string such as
+            # time [ (<replaceable>p</replaceable>) ] [ without time zone ]
+            # into types "time" and "without time zone"
+
+            # remove all the tags
+            line = re.sub("<replaceable>[^<]+</replaceable>", "", line)
+            line = re.sub("<[^>]+>", "", line)
+
+            # Drop the parts containing braces
+            for tmp in [t for tmp in line.split('[')
+                        for t in tmp.split(']') if "(" not in t]:
+                for t in tmp.split(','):
+                    t = t.strip()
+                    if not t: continue
+                    dt.add(" ".join(t.split()))
+
+        dt = list(dt)
+        dt.sort()
+        return dt
+
+    def parse_pseudos(f):
+        dt = []
+        re_start = re.compile(r'\s*<table id="datatype-pseudotypes-table">')
+        re_entry = re.compile(r'\s*<entry><type>([^<]+)</></entry>')
+        re_end = re.compile(r'\s*</table>')
+
+        f = iter(f)
+        for line in f:
+            if re_start.match(line) is not None:
+                break
+        else:
+            raise ValueError('pseudo datatypes table not found')
+
+        for line in f:
+            m = re_entry.match(line)
+            if m is not None:
+                dt.append(m.group(1))
+
+            if re_end.match(line) is not None:
+                break
+        else:
+            raise ValueError('end of pseudo datatypes table not found')
+
+        if not dt:
+            raise ValueError('pseudo datatypes not found')
+
+        return dt
+
+    def update_consts(filename, constname, content):
+        with open(filename) as f:
+            data = f.read()
+
+        # Line to start/end inserting
+        re_match = re.compile(r'^%s\s*=\s*\($.*?^\s*\)$' % constname, re.M | re.S)
+        m = re_match.search(data)
+        if not m:
+            raise ValueError('Could not find existing definition for %s' %
+                             (constname,))
+
+        new_block = format_lines(constname, content)
+        data = data[:m.start()] + new_block + data[m.end():]
+
+        with open(filename, 'w') as f:
+            f.write(data)
+
+    update_myself()

packages/wakatime/packages/pygments/lexers/_stan_builtins.py

@@ -1,18 +1,33 @@
 # -*- coding: utf-8 -*-
 """
-pygments.lexers._stan_builtins
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    pygments.lexers._stan_builtins
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-This file contains the names of functions for Stan used by
-``pygments.lexers.math.StanLexer.
+    This file contains the names of functions for Stan used by
+    ``pygments.lexers.math.StanLexer. This is for Stan language version 2.8.0.
 
-:copyright: Copyright 2013 by the Pygments team, see AUTHORS.
-:license: BSD, see LICENSE for details.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
 """
 
-KEYWORDS = ['else', 'for', 'if', 'in', 'lower', 'lp__', 'print', 'upper', 'while']
+KEYWORDS = (
+    'else',
+    'for',
+    'if',
+    'in',
+    'increment_log_prob',
+    'integrate_ode',
+    'lp__',
+    'print',
+    'reject',
+    'return',
+    'while'
+)
 
-TYPES = [   'corr_matrix',
+TYPES = (
+    'cholesky_factor_corr',
+    'cholesky_factor_cov',
+    'corr_matrix',
     'cov_matrix',
     'int',
     'matrix',
@@ -20,229 +35,376 @@ TYPES = [   'corr_matrix',
     'positive_ordered',
     'real',
     'row_vector',
+    'row_vectormatrix',
     'simplex',
     'unit_vector',
-    'vector']
+    'vector',
+    'void')
 
-FUNCTIONS = [   'Phi',
+FUNCTIONS = (
+    'Phi',
     'Phi_approx',
     'abs',
     'acos',
     'acosh',
+    'append_col',
+    'append_row',
     'asin',
     'asinh',
     'atan',
     'atan2',
     'atanh',
+    'bernoulli_ccdf_log',
     'bernoulli_cdf',
+    'bernoulli_cdf_log',
     'bernoulli_log',
     'bernoulli_logit_log',
     'bernoulli_rng',
+    'bessel_first_kind',
+    'bessel_second_kind',
+    'beta_binomial_ccdf_log',
     'beta_binomial_cdf',
+    'beta_binomial_cdf_log',
     'beta_binomial_log',
     'beta_binomial_rng',
+    'beta_ccdf_log',
     'beta_cdf',
+    'beta_cdf_log',
     'beta_log',
     'beta_rng',
     'binary_log_loss',
+    'binomial_ccdf_log',
     'binomial_cdf',
+    'binomial_cdf_log',
     'binomial_coefficient_log',
     'binomial_log',
     'binomial_logit_log',
     'binomial_rng',
     'block',
     'categorical_log',
+    'categorical_logit_log',
     'categorical_rng',
+    'cauchy_ccdf_log',
     'cauchy_cdf',
+    'cauchy_cdf_log',
     'cauchy_log',
     'cauchy_rng',
     'cbrt',
     'ceil',
+    'chi_square_ccdf_log',
+    'chi_square_cdf',
+    'chi_square_cdf_log',
     'chi_square_log',
     'chi_square_rng',
     'cholesky_decompose',
     'col',
     'cols',
+    'columns_dot_product',
+    'columns_dot_self',
     'cos',
     'cosh',
     'crossprod',
+    'csr_extract_u',
+    'csr_extract_v',
+    'csr_extract_w',
+    'csr_matrix_times_vector',
+    'csr_to_dense_matrix',
     'cumulative_sum',
     'determinant',
     'diag_matrix',
     'diag_post_multiply',
     'diag_pre_multiply',
     'diagonal',
+    'digamma',
     'dims',
     'dirichlet_log',
     'dirichlet_rng',
+    'distance',
     'dot_product',
     'dot_self',
+    'double_exponential_ccdf_log',
+    'double_exponential_cdf',
+    'double_exponential_cdf_log',
     'double_exponential_log',
     'double_exponential_rng',
     'e',
     'eigenvalues_sym',
     'eigenvectors_sym',
-    'epsilon',
     'erf',
     'erfc',
     'exp',
     'exp2',
+    'exp_mod_normal_ccdf_log',
     'exp_mod_normal_cdf',
+    'exp_mod_normal_cdf_log',
     'exp_mod_normal_log',
     'exp_mod_normal_rng',
     'expm1',
+    'exponential_ccdf_log',
     'exponential_cdf',
+    'exponential_cdf_log',
     'exponential_log',
     'exponential_rng',
     'fabs',
+    'falling_factorial',
     'fdim',
     'floor',
     'fma',
     'fmax',
     'fmin',
     'fmod',
+    'frechet_ccdf_log',
+    'frechet_cdf',
+    'frechet_cdf_log',
+    'frechet_log',
+    'frechet_rng',
+    'gamma_ccdf_log',
+    'gamma_cdf',
+    'gamma_cdf_log',
     'gamma_log',
+    'gamma_p',
+    'gamma_q',
     'gamma_rng',
+    'gaussian_dlm_obs_log',
+    'get_lp',
+    'gumbel_ccdf_log',
     'gumbel_cdf',
+    'gumbel_cdf_log',
     'gumbel_log',
     'gumbel_rng',
+    'head',
     'hypergeometric_log',
     'hypergeometric_rng',
     'hypot',
     'if_else',
     'int_step',
+    'inv',
+    'inv_chi_square_ccdf_log',
     'inv_chi_square_cdf',
+    'inv_chi_square_cdf_log',
     'inv_chi_square_log',
     'inv_chi_square_rng',
     'inv_cloglog',
+    'inv_gamma_ccdf_log',
     'inv_gamma_cdf',
+    'inv_gamma_cdf_log',
     'inv_gamma_log',
     'inv_gamma_rng',
     'inv_logit',
+    'inv_phi',
+    'inv_sqrt',
+    'inv_square',
     'inv_wishart_log',
     'inv_wishart_rng',
     'inverse',
+    'inverse_spd',
+    'is_inf',
+    'is_nan',
     'lbeta',
     'lgamma',
     'lkj_corr_cholesky_log',
     'lkj_corr_cholesky_rng',
     'lkj_corr_log',
     'lkj_corr_rng',
-    'lkj_cov_log',
     'lmgamma',
     'log',
     'log10',
     'log1m',
+    'log1m_exp',
     'log1m_inv_logit',
     'log1p',
     'log1p_exp',
     'log2',
     'log_determinant',
+    'log_diff_exp',
+    'log_falling_factorial',
     'log_inv_logit',
+    'log_mix',
+    'log_rising_factorial',
+    'log_softmax',
     'log_sum_exp',
+    'logistic_ccdf_log',
     'logistic_cdf',
+    'logistic_cdf_log',
     'logistic_log',
     'logistic_rng',
     'logit',
+    'lognormal_ccdf_log',
     'lognormal_cdf',
+    'lognormal_cdf_log',
     'lognormal_log',
     'lognormal_rng',
+    'machine_precision',
     'max',
     'mdivide_left_tri_low',
     'mdivide_right_tri_low',
     'mean',
     'min',
+    'modified_bessel_first_kind',
+    'modified_bessel_second_kind',
+    'multi_gp_cholesky_log',
+    'multi_gp_log',
     'multi_normal_cholesky_log',
+    'multi_normal_cholesky_rng',
     'multi_normal_log',
     'multi_normal_prec_log',
     'multi_normal_rng',
     'multi_student_t_log',
     'multi_student_t_rng',
-    'multinomial_cdf',
     'multinomial_log',
     'multinomial_rng',
     'multiply_log',
     'multiply_lower_tri_self_transpose',
+    'neg_binomial_2_ccdf_log',
+    'neg_binomial_2_cdf',
+    'neg_binomial_2_cdf_log',
+    'neg_binomial_2_log',
+    'neg_binomial_2_log_log',
+    'neg_binomial_2_log_rng',
+    'neg_binomial_2_rng',
+    'neg_binomial_ccdf_log',
     'neg_binomial_cdf',
+    'neg_binomial_cdf_log',
     'neg_binomial_log',
     'neg_binomial_rng',
-    'negative_epsilon',
     'negative_infinity',
+    'normal_ccdf_log',
     'normal_cdf',
+    'normal_cdf_log',
     'normal_log',
     'normal_rng',
     'not_a_number',
+    'num_elements',
     'ordered_logistic_log',
     'ordered_logistic_rng',
     'owens_t',
+    'pareto_ccdf_log',
     'pareto_cdf',
+    'pareto_cdf_log',
     'pareto_log',
     'pareto_rng',
+    'pareto_type_2_ccdf_log',
+    'pareto_type_2_cdf',
+    'pareto_type_2_cdf_log',
+    'pareto_type_2_log',
+    'pareto_type_2_rng',
     'pi',
+    'poisson_ccdf_log',
     'poisson_cdf',
+    'poisson_cdf_log',
     'poisson_log',
     'poisson_log_log',
+    'poisson_log_rng',
     'poisson_rng',
     'positive_infinity',
     'pow',
     'prod',
+    'qr_Q',
+    'qr_R',
+    'quad_form',
+    'quad_form_diag',
+    'quad_form_sym',
+    'rank',
+    'rayleigh_ccdf_log',
+    'rayleigh_cdf',
+    'rayleigh_cdf_log',
+    'rayleigh_log',
+    'rayleigh_rng',
     'rep_array',
     'rep_matrix',
     'rep_row_vector',
     'rep_vector',
+    'rising_factorial',
     'round',
     'row',
     'rows',
+    'rows_dot_product',
+    'rows_dot_self',
+    'scaled_inv_chi_square_ccdf_log',
     'scaled_inv_chi_square_cdf',
+    'scaled_inv_chi_square_cdf_log',
     'scaled_inv_chi_square_log',
     'scaled_inv_chi_square_rng',
     'sd',
+    'segment',
     'sin',
     'singular_values',
     'sinh',
     'size',
+    'skew_normal_ccdf_log',
     'skew_normal_cdf',
+    'skew_normal_cdf_log',
     'skew_normal_log',
     'skew_normal_rng',
     'softmax',
+    'sort_asc',
+    'sort_desc',
+    'sort_indices_asc',
+    'sort_indices_desc',
     'sqrt',
     'sqrt2',
     'square',
+    'squared_distance',
     'step',
+    'student_t_ccdf_log',
     'student_t_cdf',
+    'student_t_cdf_log',
     'student_t_log',
     'student_t_rng',
+    'sub_col',
+    'sub_row',
     'sum',
+    'tail',
     'tan',
     'tanh',
     'tcrossprod',
     'tgamma',
+    'to_array_1d',
+    'to_array_2d',
+    'to_matrix',
+    'to_row_vector',
+    'to_vector',
     'trace',
+    'trace_gen_quad_form',
+    'trace_quad_form',
+    'trigamma',
     'trunc',
+    'uniform_ccdf_log',
+    'uniform_cdf',
+    'uniform_cdf_log',
     'uniform_log',
     'uniform_rng',
     'variance',
+    'von_mises_log',
+    'von_mises_rng',
+    'weibull_ccdf_log',
     'weibull_cdf',
+    'weibull_cdf_log',
     'weibull_log',
     'weibull_rng',
+    'wiener_log',
     'wishart_log',
-    'wishart_rng']
+    'wishart_rng'
+)
 
-DISTRIBUTIONS = [   'bernoulli',
+DISTRIBUTIONS = (
+    'bernoulli',
     'bernoulli_logit',
     'beta',
     'beta_binomial',
     'binomial',
-    'binomial_coefficient',
     'binomial_logit',
     'categorical',
+    'categorical_logit',
     'cauchy',
     'chi_square',
     'dirichlet',
     'double_exponential',
     'exp_mod_normal',
     'exponential',
+    'frechet',
     'gamma',
+    'gaussian_dlm_obs',
     'gumbel',
     'hypergeometric',
     'inv_chi_square',
@@ -250,29 +412,37 @@ DISTRIBUTIONS = [   'bernoulli',
     'inv_wishart',
     'lkj_corr',
     'lkj_corr_cholesky',
-    'lkj_cov',
     'logistic',
     'lognormal',
+    'multi_gp',
+    'multi_gp_cholesky',
     'multi_normal',
     'multi_normal_cholesky',
     'multi_normal_prec',
     'multi_student_t',
     'multinomial',
-    'multiply',
     'neg_binomial',
+    'neg_binomial_2',
+    'neg_binomial_2_log',
     'normal',
     'ordered_logistic',
     'pareto',
+    'pareto_type_2',
     'poisson',
     'poisson_log',
+    'rayleigh',
     'scaled_inv_chi_square',
     'skew_normal',
     'student_t',
     'uniform',
+    'von_mises',
     'weibull',
-    'wishart']
+    'wiener',
+    'wishart'
+)
 
-RESERVED = [   'alignas',
+RESERVED = (
+    'alignas',
     'alignof',
     'and',
     'and_eq',
@@ -307,6 +477,7 @@ RESERVED = [   'alignas',
     'false',
     'float',
     'friend',
+    'fvar',
     'goto',
     'inline',
     'int',
@@ -327,7 +498,6 @@ RESERVED = [   'alignas',
     'register',
     'reinterpret_cast',
     'repeat',
-    'return',
     'short',
     'signed',
     'sizeof',
@@ -351,10 +521,12 @@ RESERVED = [   'alignas',
     'unsigned',
     'until',
     'using',
+    'var',
     'virtual',
     'void',
     'volatile',
     'wchar_t',
     'xor',
-    'xor_eq']
+    'xor_eq'
+)
 

packages/wakatime/packages/pygments/lexers/_stata_builtins.py

@@ -0,0 +1,419 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers._stata_builtins
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Builtins for Stata
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+
+builtins_base = (
+    "if", "else", "in", "foreach", "for", "forv", "forva",
+    "forval", "forvalu", "forvalue", "forvalues", "by", "bys",
+    "bysort", "quietly", "qui", "about", "ac",
+    "ac_7", "acprplot", "acprplot_7", "adjust", "ado", "adopath",
+    "adoupdate", "alpha", "ameans", "an", "ano", "anov", "anova",
+    "anova_estat", "anova_terms", "anovadef", "aorder", "ap", "app",
+    "appe", "appen", "append", "arch", "arch_dr", "arch_estat",
+    "arch_p", "archlm", "areg", "areg_p", "args", "arima",
+    "arima_dr", "arima_estat", "arima_p", "as", "asmprobit",
+    "asmprobit_estat", "asmprobit_lf", "asmprobit_mfx__dlg",
+    "asmprobit_p", "ass", "asse", "asser", "assert", "avplot",
+    "avplot_7", "avplots", "avplots_7", "bcskew0", "bgodfrey",
+    "binreg", "bip0_lf", "biplot", "bipp_lf", "bipr_lf",
+    "bipr_p", "biprobit", "bitest", "bitesti", "bitowt", "blogit",
+    "bmemsize", "boot", "bootsamp", "bootstrap", "bootstrap_8",
+    "boxco_l", "boxco_p", "boxcox", "boxcox_6", "boxcox_p",
+    "bprobit", "br", "break", "brier", "bro", "brow", "brows",
+    "browse", "brr", "brrstat", "bs", "bs_7", "bsampl_w",
+    "bsample", "bsample_7", "bsqreg", "bstat", "bstat_7", "bstat_8",
+    "bstrap", "bstrap_7", "ca", "ca_estat", "ca_p", "cabiplot",
+    "camat", "canon", "canon_8", "canon_8_p", "canon_estat",
+    "canon_p", "cap", "caprojection", "capt", "captu", "captur",
+    "capture", "cat", "cc", "cchart", "cchart_7", "cci",
+    "cd", "censobs_table", "centile", "cf", "char", "chdir",
+    "checkdlgfiles", "checkestimationsample", "checkhlpfiles",
+    "checksum", "chelp", "ci", "cii", "cl", "class", "classutil",
+    "clear", "cli", "clis", "clist", "clo", "clog", "clog_lf",
+    "clog_p", "clogi", "clogi_sw", "clogit", "clogit_lf",
+    "clogit_p", "clogitp", "clogl_sw", "cloglog", "clonevar",
+    "clslistarray", "cluster", "cluster_measures", "cluster_stop",
+    "cluster_tree", "cluster_tree_8", "clustermat", "cmdlog",
+    "cnr", "cnre", "cnreg", "cnreg_p", "cnreg_sw", "cnsreg",
+    "codebook", "collaps4", "collapse", "colormult_nb",
+    "colormult_nw", "compare", "compress", "conf", "confi",
+    "confir", "confirm", "conren", "cons", "const", "constr",
+    "constra", "constrai", "constrain", "constraint", "continue",
+    "contract", "copy", "copyright", "copysource", "cor", "corc",
+    "corr", "corr2data", "corr_anti", "corr_kmo", "corr_smc",
+    "corre", "correl", "correla", "correlat", "correlate",
+    "corrgram", "cou", "coun", "count", "cox", "cox_p", "cox_sw",
+    "coxbase", "coxhaz", "coxvar", "cprplot", "cprplot_7",
+    "crc", "cret", "cretu", "cretur", "creturn", "cross", "cs",
+    "cscript", "cscript_log", "csi", "ct", "ct_is", "ctset",
+    "ctst_5", "ctst_st", "cttost", "cumsp", "cumsp_7", "cumul",
+    "cusum", "cusum_7", "cutil", "d", "datasig", "datasign",
+    "datasigna", "datasignat", "datasignatu", "datasignatur",
+    "datasignature", "datetof", "db", "dbeta", "de", "dec",
+    "deco", "decod", "decode", "deff", "des", "desc", "descr",
+    "descri", "describ", "describe", "destring", "dfbeta",
+    "dfgls", "dfuller", "di", "di_g", "dir", "dirstats", "dis",
+    "discard", "disp", "disp_res", "disp_s", "displ", "displa",
+    "display", "distinct", "do", "doe", "doed", "doedi",
+    "doedit", "dotplot", "dotplot_7", "dprobit", "drawnorm",
+    "drop", "ds", "ds_util", "dstdize", "duplicates", "durbina",
+    "dwstat", "dydx", "e", "ed", "edi", "edit", "egen",
+    "eivreg", "emdef", "en", "enc", "enco", "encod", "encode",
+    "eq", "erase", "ereg", "ereg_lf", "ereg_p", "ereg_sw",
+    "ereghet", "ereghet_glf", "ereghet_glf_sh", "ereghet_gp",
+    "ereghet_ilf", "ereghet_ilf_sh", "ereghet_ip", "eret",
+    "eretu", "eretur", "ereturn", "err", "erro", "error", "est",
+    "est_cfexist", "est_cfname", "est_clickable", "est_expand",
+    "est_hold", "est_table", "est_unhold", "est_unholdok",
+    "estat", "estat_default", "estat_summ", "estat_vce_only",
+    "esti", "estimates", "etodow", "etof", "etomdy", "ex",
+    "exi", "exit", "expand", "expandcl", "fac", "fact", "facto",
+    "factor", "factor_estat", "factor_p", "factor_pca_rotated",
+    "factor_rotate", "factormat", "fcast", "fcast_compute",
+    "fcast_graph", "fdades", "fdadesc", "fdadescr", "fdadescri",
+    "fdadescrib", "fdadescribe", "fdasav", "fdasave", "fdause",
+    "fh_st", "open", "read", "close",
+    "file", "filefilter", "fillin", "find_hlp_file", "findfile",
+    "findit", "findit_7", "fit", "fl", "fli", "flis", "flist",
+    "for5_0", "form", "forma", "format", "fpredict", "frac_154",
+    "frac_adj", "frac_chk", "frac_cox", "frac_ddp", "frac_dis",
+    "frac_dv", "frac_in", "frac_mun", "frac_pp", "frac_pq",
+    "frac_pv", "frac_wgt", "frac_xo", "fracgen", "fracplot",
+    "fracplot_7", "fracpoly", "fracpred", "fron_ex", "fron_hn",
+    "fron_p", "fron_tn", "fron_tn2", "frontier", "ftodate", "ftoe",
+    "ftomdy", "ftowdate", "g", "gamhet_glf", "gamhet_gp",
+    "gamhet_ilf", "gamhet_ip", "gamma", "gamma_d2", "gamma_p",
+    "gamma_sw", "gammahet", "gdi_hexagon", "gdi_spokes", "ge",
+    "gen", "gene", "gener", "genera", "generat", "generate",
+    "genrank", "genstd", "genvmean", "gettoken", "gl", "gladder",
+    "gladder_7", "glim_l01", "glim_l02", "glim_l03", "glim_l04",
+    "glim_l05", "glim_l06", "glim_l07", "glim_l08", "glim_l09",
+    "glim_l10", "glim_l11", "glim_l12", "glim_lf", "glim_mu",
+    "glim_nw1", "glim_nw2", "glim_nw3", "glim_p", "glim_v1",
+    "glim_v2", "glim_v3", "glim_v4", "glim_v5", "glim_v6",
+    "glim_v7", "glm", "glm_6", "glm_p", "glm_sw", "glmpred", "glo",
+    "glob", "globa", "global", "glogit", "glogit_8", "glogit_p",
+    "gmeans", "gnbre_lf", "gnbreg", "gnbreg_5", "gnbreg_p",
+    "gomp_lf", "gompe_sw", "gomper_p", "gompertz", "gompertzhet",
+    "gomphet_glf", "gomphet_glf_sh", "gomphet_gp", "gomphet_ilf",
+    "gomphet_ilf_sh", "gomphet_ip", "gphdot", "gphpen",
+    "gphprint", "gprefs", "gprobi_p", "gprobit", "gprobit_8", "gr",
+    "gr7", "gr_copy", "gr_current", "gr_db", "gr_describe",
+    "gr_dir", "gr_draw", "gr_draw_replay", "gr_drop", "gr_edit",
+    "gr_editviewopts", "gr_example", "gr_example2", "gr_export",
+    "gr_print", "gr_qscheme", "gr_query", "gr_read", "gr_rename",
+    "gr_replay", "gr_save", "gr_set", "gr_setscheme", "gr_table",
+    "gr_undo", "gr_use", "graph", "graph7", "grebar", "greigen",
+    "greigen_7", "greigen_8", "grmeanby", "grmeanby_7",
+    "gs_fileinfo", "gs_filetype", "gs_graphinfo", "gs_stat",
+    "gsort", "gwood", "h", "hadimvo", "hareg", "hausman",
+    "haver", "he", "heck_d2", "heckma_p", "heckman", "heckp_lf",
+    "heckpr_p", "heckprob", "hel", "help", "hereg", "hetpr_lf",
+    "hetpr_p", "hetprob", "hettest", "hexdump", "hilite",
+    "hist", "hist_7", "histogram", "hlogit", "hlu", "hmeans",
+    "hotel", "hotelling", "hprobit", "hreg", "hsearch", "icd9",
+    "icd9_ff", "icd9p", "iis", "impute", "imtest", "inbase",
+    "include", "inf", "infi", "infil", "infile", "infix", "inp",
+    "inpu", "input", "ins", "insheet", "insp", "inspe",
+    "inspec", "inspect", "integ", "inten", "intreg", "intreg_7",
+    "intreg_p", "intrg2_ll", "intrg_ll", "intrg_ll2", "ipolate",
+    "iqreg", "ir", "irf", "irf_create", "irfm", "iri", "is_svy",
+    "is_svysum", "isid", "istdize", "ivprob_1_lf", "ivprob_lf",
+    "ivprobit", "ivprobit_p", "ivreg", "ivreg_footnote",
+    "ivtob_1_lf", "ivtob_lf", "ivtobit", "ivtobit_p", "jackknife",
+    "jacknife", "jknife", "jknife_6", "jknife_8", "jkstat",
+    "joinby", "kalarma1", "kap", "kap_3", "kapmeier", "kappa",
+    "kapwgt", "kdensity", "kdensity_7", "keep", "ksm", "ksmirnov",
+    "ktau", "kwallis", "l", "la", "lab", "labe", "label",
+    "labelbook", "ladder", "levels", "levelsof", "leverage",
+    "lfit", "lfit_p", "li", "lincom", "line", "linktest",
+    "lis", "list", "lloghet_glf", "lloghet_glf_sh", "lloghet_gp",
+    "lloghet_ilf", "lloghet_ilf_sh", "lloghet_ip", "llogi_sw",
+    "llogis_p", "llogist", "llogistic", "llogistichet",
+    "lnorm_lf", "lnorm_sw", "lnorma_p", "lnormal", "lnormalhet",
+    "lnormhet_glf", "lnormhet_glf_sh", "lnormhet_gp",
+    "lnormhet_ilf", "lnormhet_ilf_sh", "lnormhet_ip", "lnskew0",
+    "loadingplot", "loc", "loca", "local", "log", "logi",
+    "logis_lf", "logistic", "logistic_p", "logit", "logit_estat",
+    "logit_p", "loglogs", "logrank", "loneway", "lookfor",
+    "lookup", "lowess", "lowess_7", "lpredict", "lrecomp", "lroc",
+    "lroc_7", "lrtest", "ls", "lsens", "lsens_7", "lsens_x",
+    "lstat", "ltable", "ltable_7", "ltriang", "lv", "lvr2plot",
+    "lvr2plot_7", "m", "ma", "mac", "macr", "macro", "makecns",
+    "man", "manova", "manova_estat", "manova_p", "manovatest",
+    "mantel", "mark", "markin", "markout", "marksample", "mat",
+    "mat_capp", "mat_order", "mat_put_rr", "mat_rapp", "mata",
+    "mata_clear", "mata_describe", "mata_drop", "mata_matdescribe",
+    "mata_matsave", "mata_matuse", "mata_memory", "mata_mlib",
+    "mata_mosave", "mata_rename", "mata_which", "matalabel",
+    "matcproc", "matlist", "matname", "matr", "matri",
+    "matrix", "matrix_input__dlg", "matstrik", "mcc", "mcci",
+    "md0_", "md1_", "md1debug_", "md2_", "md2debug_", "mds",
+    "mds_estat", "mds_p", "mdsconfig", "mdslong", "mdsmat",
+    "mdsshepard", "mdytoe", "mdytof", "me_derd", "mean",
+    "means", "median", "memory", "memsize", "meqparse", "mer",
+    "merg", "merge", "mfp", "mfx", "mhelp", "mhodds", "minbound",
+    "mixed_ll", "mixed_ll_reparm", "mkassert", "mkdir",
+    "mkmat", "mkspline", "ml", "ml_5", "ml_adjs", "ml_bhhhs",
+    "ml_c_d", "ml_check", "ml_clear", "ml_cnt", "ml_debug",
+    "ml_defd", "ml_e0", "ml_e0_bfgs", "ml_e0_cycle", "ml_e0_dfp",
+    "ml_e0i", "ml_e1", "ml_e1_bfgs", "ml_e1_bhhh", "ml_e1_cycle",
+    "ml_e1_dfp", "ml_e2", "ml_e2_cycle", "ml_ebfg0", "ml_ebfr0",
+    "ml_ebfr1", "ml_ebh0q", "ml_ebhh0", "ml_ebhr0", "ml_ebr0i",
+    "ml_ecr0i", "ml_edfp0", "ml_edfr0", "ml_edfr1", "ml_edr0i",
+    "ml_eds", "ml_eer0i", "ml_egr0i", "ml_elf", "ml_elf_bfgs",
+    "ml_elf_bhhh", "ml_elf_cycle", "ml_elf_dfp", "ml_elfi",
+    "ml_elfs", "ml_enr0i", "ml_enrr0", "ml_erdu0", "ml_erdu0_bfgs",
+    "ml_erdu0_bhhh", "ml_erdu0_bhhhq", "ml_erdu0_cycle",
+    "ml_erdu0_dfp", "ml_erdu0_nrbfgs", "ml_exde", "ml_footnote",
+    "ml_geqnr", "ml_grad0", "ml_graph", "ml_hbhhh", "ml_hd0",
+    "ml_hold", "ml_init", "ml_inv", "ml_log", "ml_max",
+    "ml_mlout", "ml_mlout_8", "ml_model", "ml_nb0", "ml_opt",
+    "ml_p", "ml_plot", "ml_query", "ml_rdgrd", "ml_repor",
+    "ml_s_e", "ml_score", "ml_searc", "ml_technique", "ml_unhold",
+    "mleval", "mlf_", "mlmatbysum", "mlmatsum", "mlog", "mlogi",
+    "mlogit", "mlogit_footnote", "mlogit_p", "mlopts", "mlsum",
+    "mlvecsum", "mnl0_", "mor", "more", "mov", "move", "mprobit",
+    "mprobit_lf", "mprobit_p", "mrdu0_", "mrdu1_", "mvdecode",
+    "mvencode", "mvreg", "mvreg_estat", "n", "nbreg",
+    "nbreg_al", "nbreg_lf", "nbreg_p", "nbreg_sw", "nestreg", "net",
+    "newey", "newey_7", "newey_p", "news", "nl", "nl_7", "nl_9",
+    "nl_9_p", "nl_p", "nl_p_7", "nlcom", "nlcom_p", "nlexp2",
+    "nlexp2_7", "nlexp2a", "nlexp2a_7", "nlexp3", "nlexp3_7",
+    "nlgom3", "nlgom3_7", "nlgom4", "nlgom4_7", "nlinit", "nllog3",
+    "nllog3_7", "nllog4", "nllog4_7", "nlog_rd", "nlogit",
+    "nlogit_p", "nlogitgen", "nlogittree", "nlpred", "no",
+    "nobreak", "noi", "nois", "noisi", "noisil", "noisily", "note",
+    "notes", "notes_dlg", "nptrend", "numlabel", "numlist", "odbc",
+    "old_ver", "olo", "olog", "ologi", "ologi_sw", "ologit",
+    "ologit_p", "ologitp", "on", "one", "onew", "onewa", "oneway",
+    "op_colnm", "op_comp", "op_diff", "op_inv", "op_str", "opr",
+    "opro", "oprob", "oprob_sw", "oprobi", "oprobi_p", "oprobit",
+    "oprobitp", "opts_exclusive", "order", "orthog", "orthpoly",
+    "ou", "out", "outf", "outfi", "outfil", "outfile", "outs",
+    "outsh", "outshe", "outshee", "outsheet", "ovtest", "pac",
+    "pac_7", "palette", "parse", "parse_dissim", "pause", "pca",
+    "pca_8", "pca_display", "pca_estat", "pca_p", "pca_rotate",
+    "pcamat", "pchart", "pchart_7", "pchi", "pchi_7", "pcorr",
+    "pctile", "pentium", "pergram", "pergram_7", "permute",
+    "permute_8", "personal", "peto_st", "pkcollapse", "pkcross",
+    "pkequiv", "pkexamine", "pkexamine_7", "pkshape", "pksumm",
+    "pksumm_7", "pl", "plo", "plot", "plugin", "pnorm",
+    "pnorm_7", "poisgof", "poiss_lf", "poiss_sw", "poisso_p",
+    "poisson", "poisson_estat", "post", "postclose", "postfile",
+    "postutil", "pperron", "pr", "prais", "prais_e", "prais_e2",
+    "prais_p", "predict", "predictnl", "preserve", "print",
+    "pro", "prob", "probi", "probit", "probit_estat", "probit_p",
+    "proc_time", "procoverlay", "procrustes", "procrustes_estat",
+    "procrustes_p", "profiler", "prog", "progr", "progra",
+    "program", "prop", "proportion", "prtest", "prtesti", "pwcorr",
+    "pwd", "q", "s", "qby", "qbys", "qchi", "qchi_7", "qladder",
+    "qladder_7", "qnorm", "qnorm_7", "qqplot", "qqplot_7", "qreg",
+    "qreg_c", "qreg_p", "qreg_sw", "qu", "quadchk", "quantile",
+    "quantile_7", "que", "quer", "query", "range", "ranksum",
+    "ratio", "rchart", "rchart_7", "rcof", "recast", "reclink",
+    "recode", "reg", "reg3", "reg3_p", "regdw", "regr", "regre",
+    "regre_p2", "regres", "regres_p", "regress", "regress_estat",
+    "regriv_p", "remap", "ren", "rena", "renam", "rename",
+    "renpfix", "repeat", "replace", "report", "reshape",
+    "restore", "ret", "retu", "retur", "return", "rm", "rmdir",
+    "robvar", "roccomp", "roccomp_7", "roccomp_8", "rocf_lf",
+    "rocfit", "rocfit_8", "rocgold", "rocplot", "rocplot_7",
+    "roctab", "roctab_7", "rolling", "rologit", "rologit_p",
+    "rot", "rota", "rotat", "rotate", "rotatemat", "rreg",
+    "rreg_p", "ru", "run", "runtest", "rvfplot", "rvfplot_7",
+    "rvpplot", "rvpplot_7", "sa", "safesum", "sample",
+    "sampsi", "sav", "save", "savedresults", "saveold", "sc",
+    "sca", "scal", "scala", "scalar", "scatter", "scm_mine",
+    "sco", "scob_lf", "scob_p", "scobi_sw", "scobit", "scor",
+    "score", "scoreplot", "scoreplot_help", "scree", "screeplot",
+    "screeplot_help", "sdtest", "sdtesti", "se", "search",
+    "separate", "seperate", "serrbar", "serrbar_7", "serset", "set",
+    "set_defaults", "sfrancia", "sh", "she", "shel", "shell",
+    "shewhart", "shewhart_7", "signestimationsample", "signrank",
+    "signtest", "simul", "simul_7", "simulate", "simulate_8",
+    "sktest", "sleep", "slogit", "slogit_d2", "slogit_p", "smooth",
+    "snapspan", "so", "sor", "sort", "spearman", "spikeplot",
+    "spikeplot_7", "spikeplt", "spline_x", "split", "sqreg",
+    "sqreg_p", "sret", "sretu", "sretur", "sreturn", "ssc", "st",
+    "st_ct", "st_hc", "st_hcd", "st_hcd_sh", "st_is", "st_issys",
+    "st_note", "st_promo", "st_set", "st_show", "st_smpl",
+    "st_subid", "stack", "statsby", "statsby_8", "stbase", "stci",
+    "stci_7", "stcox", "stcox_estat", "stcox_fr", "stcox_fr_ll",
+    "stcox_p", "stcox_sw", "stcoxkm", "stcoxkm_7", "stcstat",
+    "stcurv", "stcurve", "stcurve_7", "stdes", "stem", "stepwise",
+    "stereg", "stfill", "stgen", "stir", "stjoin", "stmc", "stmh",
+    "stphplot", "stphplot_7", "stphtest", "stphtest_7",
+    "stptime", "strate", "strate_7", "streg", "streg_sw", "streset",
+    "sts", "sts_7", "stset", "stsplit", "stsum", "sttocc",
+    "sttoct", "stvary", "stweib", "su", "suest", "suest_8",
+    "sum", "summ", "summa", "summar", "summari", "summariz",
+    "summarize", "sunflower", "sureg", "survcurv", "survsum",
+    "svar", "svar_p", "svmat", "svy", "svy_disp", "svy_dreg",
+    "svy_est", "svy_est_7", "svy_estat", "svy_get", "svy_gnbreg_p",
+    "svy_head", "svy_header", "svy_heckman_p", "svy_heckprob_p",
+    "svy_intreg_p", "svy_ivreg_p", "svy_logistic_p", "svy_logit_p",
+    "svy_mlogit_p", "svy_nbreg_p", "svy_ologit_p", "svy_oprobit_p",
+    "svy_poisson_p", "svy_probit_p", "svy_regress_p", "svy_sub",
+    "svy_sub_7", "svy_x", "svy_x_7", "svy_x_p", "svydes",
+    "svydes_8", "svygen", "svygnbreg", "svyheckman", "svyheckprob",
+    "svyintreg", "svyintreg_7", "svyintrg", "svyivreg", "svylc",
+    "svylog_p", "svylogit", "svymarkout", "svymarkout_8",
+    "svymean", "svymlog", "svymlogit", "svynbreg", "svyolog",
+    "svyologit", "svyoprob", "svyoprobit", "svyopts",
+    "svypois", "svypois_7", "svypoisson", "svyprobit", "svyprobt",
+    "svyprop", "svyprop_7", "svyratio", "svyreg", "svyreg_p",
+    "svyregress", "svyset", "svyset_7", "svyset_8", "svytab",
+    "svytab_7", "svytest", "svytotal", "sw", "sw_8", "swcnreg",
+    "swcox", "swereg", "swilk", "swlogis", "swlogit",
+    "swologit", "swoprbt", "swpois", "swprobit", "swqreg",
+    "swtobit", "swweib", "symmetry", "symmi", "symplot",
+    "symplot_7", "syntax", "sysdescribe", "sysdir", "sysuse",
+    "szroeter", "ta", "tab", "tab1", "tab2", "tab_or", "tabd",
+    "tabdi", "tabdis", "tabdisp", "tabi", "table", "tabodds",
+    "tabodds_7", "tabstat", "tabu", "tabul", "tabula", "tabulat",
+    "tabulate", "te", "tempfile", "tempname", "tempvar", "tes",
+    "test", "testnl", "testparm", "teststd", "tetrachoric",
+    "time_it", "timer", "tis", "tob", "tobi", "tobit", "tobit_p",
+    "tobit_sw", "token", "tokeni", "tokeniz", "tokenize",
+    "tostring", "total", "translate", "translator", "transmap",
+    "treat_ll", "treatr_p", "treatreg", "trim", "trnb_cons",
+    "trnb_mean", "trpoiss_d2", "trunc_ll", "truncr_p", "truncreg",
+    "tsappend", "tset", "tsfill", "tsline", "tsline_ex",
+    "tsreport", "tsrevar", "tsrline", "tsset", "tssmooth",
+    "tsunab", "ttest", "ttesti", "tut_chk", "tut_wait", "tutorial",
+    "tw", "tware_st", "two", "twoway", "twoway__fpfit_serset",
+    "twoway__function_gen", "twoway__histogram_gen",
+    "twoway__ipoint_serset", "twoway__ipoints_serset",
+    "twoway__kdensity_gen", "twoway__lfit_serset",
+    "twoway__normgen_gen", "twoway__pci_serset",
+    "twoway__qfit_serset", "twoway__scatteri_serset",
+    "twoway__sunflower_gen", "twoway_ksm_serset", "ty", "typ",
+    "type", "typeof", "u", "unab", "unabbrev", "unabcmd",
+    "update", "us", "use", "uselabel", "var", "var_mkcompanion",
+    "var_p", "varbasic", "varfcast", "vargranger", "varirf",
+    "varirf_add", "varirf_cgraph", "varirf_create", "varirf_ctable",
+    "varirf_describe", "varirf_dir", "varirf_drop", "varirf_erase",
+    "varirf_graph", "varirf_ograph", "varirf_rename", "varirf_set",
+    "varirf_table", "varlist", "varlmar", "varnorm", "varsoc",
+    "varstable", "varstable_w", "varstable_w2", "varwle",
+    "vce", "vec", "vec_fevd", "vec_mkphi", "vec_p", "vec_p_w",
+    "vecirf_create", "veclmar", "veclmar_w", "vecnorm",
+    "vecnorm_w", "vecrank", "vecstable", "verinst", "vers",
+    "versi", "versio", "version", "view", "viewsource", "vif",
+    "vwls", "wdatetof", "webdescribe", "webseek", "webuse",
+    "weib1_lf", "weib2_lf", "weib_lf", "weib_lf0", "weibhet_glf",
+    "weibhet_glf_sh", "weibhet_glfa", "weibhet_glfa_sh",
+    "weibhet_gp", "weibhet_ilf", "weibhet_ilf_sh", "weibhet_ilfa",
+    "weibhet_ilfa_sh", "weibhet_ip", "weibu_sw", "weibul_p",
+    "weibull", "weibull_c", "weibull_s", "weibullhet",
+    "wh", "whelp", "whi", "which", "whil", "while", "wilc_st",
+    "wilcoxon", "win", "wind", "windo", "window", "winexec",
+    "wntestb", "wntestb_7", "wntestq", "xchart", "xchart_7",
+    "xcorr", "xcorr_7", "xi", "xi_6", "xmlsav", "xmlsave",
+    "xmluse", "xpose", "xsh", "xshe", "xshel", "xshell",
+    "xt_iis", "xt_tis", "xtab_p", "xtabond", "xtbin_p",
+    "xtclog", "xtcloglog", "xtcloglog_8", "xtcloglog_d2",
+    "xtcloglog_pa_p", "xtcloglog_re_p", "xtcnt_p", "xtcorr",
+    "xtdata", "xtdes", "xtfront_p", "xtfrontier", "xtgee",
+    "xtgee_elink", "xtgee_estat", "xtgee_makeivar", "xtgee_p",
+    "xtgee_plink", "xtgls", "xtgls_p", "xthaus", "xthausman",
+    "xtht_p", "xthtaylor", "xtile", "xtint_p", "xtintreg",
+    "xtintreg_8", "xtintreg_d2", "xtintreg_p", "xtivp_1",
+    "xtivp_2", "xtivreg", "xtline", "xtline_ex", "xtlogit",
+    "xtlogit_8", "xtlogit_d2", "xtlogit_fe_p", "xtlogit_pa_p",
+    "xtlogit_re_p", "xtmixed", "xtmixed_estat", "xtmixed_p",
+    "xtnb_fe", "xtnb_lf", "xtnbreg", "xtnbreg_pa_p",
+    "xtnbreg_refe_p", "xtpcse", "xtpcse_p", "xtpois", "xtpoisson",
+    "xtpoisson_d2", "xtpoisson_pa_p", "xtpoisson_refe_p", "xtpred",
+    "xtprobit", "xtprobit_8", "xtprobit_d2", "xtprobit_re_p",
+    "xtps_fe", "xtps_lf", "xtps_ren", "xtps_ren_8", "xtrar_p",
+    "xtrc", "xtrc_p", "xtrchh", "xtrefe_p", "xtreg", "xtreg_be",
+    "xtreg_fe", "xtreg_ml", "xtreg_pa_p", "xtreg_re",
+    "xtregar", "xtrere_p", "xtset", "xtsf_ll", "xtsf_llti",
+    "xtsum", "xttab", "xttest0", "xttobit", "xttobit_8",
+    "xttobit_p", "xttrans", "yx", "yxview__barlike_draw",
+    "yxview_area_draw", "yxview_bar_draw", "yxview_dot_draw",
+    "yxview_dropline_draw", "yxview_function_draw",
+    "yxview_iarrow_draw", "yxview_ilabels_draw",
+    "yxview_normal_draw", "yxview_pcarrow_draw",
+    "yxview_pcbarrow_draw", "yxview_pccapsym_draw",
+    "yxview_pcscatter_draw", "yxview_pcspike_draw",
+    "yxview_rarea_draw", "yxview_rbar_draw", "yxview_rbarm_draw",
+    "yxview_rcap_draw", "yxview_rcapsym_draw",
+    "yxview_rconnected_draw", "yxview_rline_draw",
+    "yxview_rscatter_draw", "yxview_rspike_draw",
+    "yxview_spike_draw", "yxview_sunflower_draw", "zap_s", "zinb",
+    "zinb_llf", "zinb_plf", "zip", "zip_llf", "zip_p", "zip_plf",
+    "zt_ct_5", "zt_hc_5", "zt_hcd_5", "zt_is_5", "zt_iss_5",
+    "zt_sho_5", "zt_smp_5", "ztbase_5", "ztcox_5", "ztdes_5",
+    "ztereg_5", "ztfill_5", "ztgen_5", "ztir_5", "ztjoin_5", "ztnb",
+    "ztnb_p", "ztp", "ztp_p", "zts_5", "ztset_5", "ztspli_5",
+    "ztsum_5", "zttoct_5", "ztvary_5", "ztweib_5"
+)
+
+builtins_functions = (
+    "Cdhms", "Chms", "Clock", "Cmdyhms", "Cofc", "Cofd", "F",
+    "Fden", "Ftail", "I", "J", "_caller", "abbrev", "abs", "acos",
+    "acosh", "asin", "asinh", "atan", "atan2", "atanh",
+    "autocode", "betaden", "binomial", "binomialp", "binomialtail",
+    "binormal", "bofd", "byteorder", "c", "ceil", "char",
+    "chi2", "chi2den", "chi2tail", "cholesky", "chop", "clip",
+    "clock", "cloglog", "cofC", "cofd", "colnumb", "colsof", "comb",
+    "cond", "corr", "cos", "cosh", "d", "daily", "date", "day",
+    "det", "dgammapda", "dgammapdada", "dgammapdadx", "dgammapdx",
+    "dgammapdxdx", "dhms", "diag", "diag0cnt", "digamma",
+    "dofC", "dofb", "dofc", "dofh", "dofm", "dofq", "dofw",
+    "dofy", "dow", "doy", "dunnettprob", "e", "el", "epsdouble",
+    "epsfloat", "exp", "fileexists", "fileread", "filereaderror",
+    "filewrite", "float", "floor", "fmtwidth", "gammaden",
+    "gammap", "gammaptail", "get", "group", "h", "hadamard",
+    "halfyear", "halfyearly", "has_eprop", "hh", "hhC", "hms",
+    "hofd", "hours", "hypergeometric", "hypergeometricp", "ibeta",
+    "ibetatail", "index", "indexnot", "inlist", "inrange", "int",
+    "inv", "invF", "invFtail", "invbinomial", "invbinomialtail",
+    "invchi2", "invchi2tail", "invcloglog", "invdunnettprob",
+    "invgammap", "invgammaptail", "invibeta", "invibetatail",
+    "invlogit", "invnFtail", "invnbinomial", "invnbinomialtail",
+    "invnchi2", "invnchi2tail", "invnibeta", "invnorm", "invnormal",
+    "invnttail", "invpoisson", "invpoissontail", "invsym", "invt",
+    "invttail", "invtukeyprob", "irecode", "issym", "issymmetric",
+    "itrim", "length", "ln", "lnfact", "lnfactorial", "lngamma",
+    "lnnormal", "lnnormalden", "log", "log10", "logit", "lower",
+    "ltrim", "m", "match", "matmissing", "matrix", "matuniform",
+    "max", "maxbyte", "maxdouble", "maxfloat", "maxint", "maxlong",
+    "mdy", "mdyhms", "mi", "min", "minbyte", "mindouble",
+    "minfloat", "minint", "minlong", "minutes", "missing", "mm",
+    "mmC", "mod", "mofd", "month", "monthly", "mreldif",
+    "msofhours", "msofminutes", "msofseconds", "nF", "nFden",
+    "nFtail", "nbetaden", "nbinomial", "nbinomialp", "nbinomialtail",
+    "nchi2", "nchi2den", "nchi2tail", "nibeta", "norm", "normal",
+    "normalden", "normd", "npnF", "npnchi2", "npnt", "nt", "ntden",
+    "nttail", "nullmat", "plural", "poisson", "poissonp",
+    "poissontail", "proper", "q", "qofd", "quarter", "quarterly",
+    "r", "rbeta", "rbinomial", "rchi2", "real", "recode", "regexm",
+    "regexr", "regexs", "reldif", "replay", "return", "reverse",
+    "rgamma", "rhypergeometric", "rnbinomial", "rnormal", "round",
+    "rownumb", "rowsof", "rpoisson", "rt", "rtrim", "runiform", "s",
+    "scalar", "seconds", "sign", "sin", "sinh", "smallestdouble",
+    "soundex", "soundex_nara", "sqrt", "ss", "ssC", "strcat",
+    "strdup", "string", "strlen", "strlower", "strltrim", "strmatch",
+    "strofreal", "strpos", "strproper", "strreverse", "strrtrim",
+    "strtoname", "strtrim", "strupper", "subinstr", "subinword",
+    "substr", "sum", "sweep", "syminv", "t", "tC", "tan", "tanh",
+    "tc", "td", "tden", "th", "tin", "tm", "tq", "trace",
+    "trigamma", "trim", "trunc", "ttail", "tukeyprob", "tw",
+    "twithin", "uniform", "upper", "vec", "vecdiag", "w", "week",
+    "weekly", "wofd", "word", "wordcount", "year", "yearly",
+    "yh", "ym", "yofd", "yq", "yw"
+)
+
+

packages/wakatime/packages/pygments/lexers/actionscript.py

@@ -0,0 +1,240 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.actionscript
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for ActionScript and MXML.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, using, this, words, default
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['ActionScriptLexer', 'ActionScript3Lexer', 'MxmlLexer']
+
+
+class ActionScriptLexer(RegexLexer):
+    """
+    For ActionScript source code.
+
+    .. versionadded:: 0.9
+    """
+
+    name = 'ActionScript'
+    aliases = ['as', 'actionscript']
+    filenames = ['*.as']
+    mimetypes = ['application/x-actionscript', 'text/x-actionscript',
+                 'text/actionscript']
+
+    flags = re.DOTALL
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+            (r'//.*?\n', Comment.Single),
+            (r'/\*.*?\*/', Comment.Multiline),
+            (r'/(\\\\|\\/|[^/\n])*/[gim]*', String.Regex),
+            (r'[~^*!%&<>|+=:;,/?\\-]+', Operator),
+            (r'[{}\[\]();.]+', Punctuation),
+            (words((
+                'case', 'default', 'for', 'each', 'in', 'while', 'do', 'break',
+                'return', 'continue', 'if', 'else', 'throw', 'try', 'catch',
+                'var', 'with', 'new', 'typeof', 'arguments', 'instanceof', 'this',
+                'switch'), suffix=r'\b'),
+             Keyword),
+            (words((
+                'class', 'public', 'final', 'internal', 'native', 'override', 'private',
+                'protected', 'static', 'import', 'extends', 'implements', 'interface',
+                'intrinsic', 'return', 'super', 'dynamic', 'function', 'const', 'get',
+                'namespace', 'package', 'set'), suffix=r'\b'),
+             Keyword.Declaration),
+            (r'(true|false|null|NaN|Infinity|-Infinity|undefined|Void)\b',
+             Keyword.Constant),
+            (words((
+                'Accessibility', 'AccessibilityProperties', 'ActionScriptVersion',
+                'ActivityEvent', 'AntiAliasType', 'ApplicationDomain', 'AsBroadcaster', 'Array',
+                'AsyncErrorEvent', 'AVM1Movie', 'BevelFilter', 'Bitmap', 'BitmapData',
+                'BitmapDataChannel', 'BitmapFilter', 'BitmapFilterQuality', 'BitmapFilterType',
+                'BlendMode', 'BlurFilter', 'Boolean', 'ByteArray', 'Camera', 'Capabilities', 'CapsStyle',
+                'Class', 'Color', 'ColorMatrixFilter', 'ColorTransform', 'ContextMenu',
+                'ContextMenuBuiltInItems', 'ContextMenuEvent', 'ContextMenuItem',
+                'ConvultionFilter', 'CSMSettings', 'DataEvent', 'Date', 'DefinitionError',
+                'DeleteObjectSample', 'Dictionary', 'DisplacmentMapFilter', 'DisplayObject',
+                'DisplacmentMapFilterMode', 'DisplayObjectContainer', 'DropShadowFilter',
+                'Endian', 'EOFError', 'Error', 'ErrorEvent', 'EvalError', 'Event', 'EventDispatcher',
+                'EventPhase', 'ExternalInterface', 'FileFilter', 'FileReference',
+                'FileReferenceList', 'FocusDirection', 'FocusEvent', 'Font', 'FontStyle', 'FontType',
+                'FrameLabel', 'FullScreenEvent', 'Function', 'GlowFilter', 'GradientBevelFilter',
+                'GradientGlowFilter', 'GradientType', 'Graphics', 'GridFitType', 'HTTPStatusEvent',
+                'IBitmapDrawable', 'ID3Info', 'IDataInput', 'IDataOutput', 'IDynamicPropertyOutput'
+                'IDynamicPropertyWriter', 'IEventDispatcher', 'IExternalizable',
+                'IllegalOperationError', 'IME', 'IMEConversionMode', 'IMEEvent', 'int',
+                'InteractiveObject', 'InterpolationMethod', 'InvalidSWFError', 'InvokeEvent',
+                'IOError', 'IOErrorEvent', 'JointStyle', 'Key', 'Keyboard', 'KeyboardEvent', 'KeyLocation',
+                'LineScaleMode', 'Loader', 'LoaderContext', 'LoaderInfo', 'LoadVars', 'LocalConnection',
+                'Locale', 'Math', 'Matrix', 'MemoryError', 'Microphone', 'MorphShape', 'Mouse', 'MouseEvent',
+                'MovieClip', 'MovieClipLoader', 'Namespace', 'NetConnection', 'NetStatusEvent',
+                'NetStream', 'NewObjectSample', 'Number', 'Object', 'ObjectEncoding', 'PixelSnapping',
+                'Point', 'PrintJob', 'PrintJobOptions', 'PrintJobOrientation', 'ProgressEvent', 'Proxy',
+                'QName', 'RangeError', 'Rectangle', 'ReferenceError', 'RegExp', 'Responder', 'Sample',
+                'Scene', 'ScriptTimeoutError', 'Security', 'SecurityDomain', 'SecurityError',
+                'SecurityErrorEvent', 'SecurityPanel', 'Selection', 'Shape', 'SharedObject',
+                'SharedObjectFlushStatus', 'SimpleButton', 'Socket', 'Sound', 'SoundChannel',
+                'SoundLoaderContext', 'SoundMixer', 'SoundTransform', 'SpreadMethod', 'Sprite',
+                'StackFrame', 'StackOverflowError', 'Stage', 'StageAlign', 'StageDisplayState',
+                'StageQuality', 'StageScaleMode', 'StaticText', 'StatusEvent', 'String', 'StyleSheet',
+                'SWFVersion', 'SyncEvent', 'SyntaxError', 'System', 'TextColorType', 'TextField',
+                'TextFieldAutoSize', 'TextFieldType', 'TextFormat', 'TextFormatAlign',
+                'TextLineMetrics', 'TextRenderer', 'TextSnapshot', 'Timer', 'TimerEvent', 'Transform',
+                'TypeError', 'uint', 'URIError', 'URLLoader', 'URLLoaderDataFormat', 'URLRequest',
+                'URLRequestHeader', 'URLRequestMethod', 'URLStream', 'URLVariabeles', 'VerifyError',
+                'Video', 'XML', 'XMLDocument', 'XMLList', 'XMLNode', 'XMLNodeType', 'XMLSocket',
+                'XMLUI'), suffix=r'\b'),
+             Name.Builtin),
+            (words((
+                'decodeURI', 'decodeURIComponent', 'encodeURI', 'escape', 'eval', 'isFinite', 'isNaN',
+                'isXMLName', 'clearInterval', 'fscommand', 'getTimer', 'getURL', 'getVersion',
+                'parseFloat', 'parseInt', 'setInterval', 'trace', 'updateAfterEvent',
+                'unescape'), suffix=r'\b'),
+             Name.Function),
+            (r'[$a-zA-Z_]\w*', Name.Other),
+            (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+            (r'0x[0-9a-f]+', Number.Hex),
+            (r'[0-9]+', Number.Integer),
+            (r'"(\\\\|\\"|[^"])*"', String.Double),
+            (r"'(\\\\|\\'|[^'])*'", String.Single),
+        ]
+    }
+
+
+class ActionScript3Lexer(RegexLexer):
+    """
+    For ActionScript 3 source code.
+
+    .. versionadded:: 0.11
+    """
+
+    name = 'ActionScript 3'
+    aliases = ['as3', 'actionscript3']
+    filenames = ['*.as']
+    mimetypes = ['application/x-actionscript3', 'text/x-actionscript3',
+                 'text/actionscript3']
+
+    identifier = r'[$a-zA-Z_]\w*'
+    typeidentifier = identifier + '(?:\.<\w+>)?'
+
+    flags = re.DOTALL | re.MULTILINE
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+            (r'(function\s+)(' + identifier + r')(\s*)(\()',
+             bygroups(Keyword.Declaration, Name.Function, Text, Operator),
+             'funcparams'),
+            (r'(var|const)(\s+)(' + identifier + r')(\s*)(:)(\s*)(' +
+             typeidentifier + r')',
+             bygroups(Keyword.Declaration, Text, Name, Text, Punctuation, Text,
+                      Keyword.Type)),
+            (r'(import|package)(\s+)((?:' + identifier + r'|\.)+)(\s*)',
+             bygroups(Keyword, Text, Name.Namespace, Text)),
+            (r'(new)(\s+)(' + typeidentifier + r')(\s*)(\()',
+             bygroups(Keyword, Text, Keyword.Type, Text, Operator)),
+            (r'//.*?\n', Comment.Single),
+            (r'/\*.*?\*/', Comment.Multiline),
+            (r'/(\\\\|\\/|[^\n])*/[gisx]*', String.Regex),
+            (r'(\.)(' + identifier + r')', bygroups(Operator, Name.Attribute)),
+            (r'(case|default|for|each|in|while|do|break|return|continue|if|else|'
+             r'throw|try|catch|with|new|typeof|arguments|instanceof|this|'
+             r'switch|import|include|as|is)\b',
+             Keyword),
+            (r'(class|public|final|internal|native|override|private|protected|'
+             r'static|import|extends|implements|interface|intrinsic|return|super|'
+             r'dynamic|function|const|get|namespace|package|set)\b',
+             Keyword.Declaration),
+            (r'(true|false|null|NaN|Infinity|-Infinity|undefined|void)\b',
+             Keyword.Constant),
+            (r'(decodeURI|decodeURIComponent|encodeURI|escape|eval|isFinite|isNaN|'
+             r'isXMLName|clearInterval|fscommand|getTimer|getURL|getVersion|'
+             r'isFinite|parseFloat|parseInt|setInterval|trace|updateAfterEvent|'
+             r'unescape)\b', Name.Function),
+            (identifier, Name),
+            (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+            (r'0x[0-9a-f]+', Number.Hex),
+            (r'[0-9]+', Number.Integer),
+            (r'"(\\\\|\\"|[^"])*"', String.Double),
+            (r"'(\\\\|\\'|[^'])*'", String.Single),
+            (r'[~^*!%&<>|+=:;,/?\\{}\[\]().-]+', Operator),
+        ],
+        'funcparams': [
+            (r'\s+', Text),
+            (r'(\s*)(\.\.\.)?(' + identifier + r')(\s*)(:)(\s*)(' +
+             typeidentifier + r'|\*)(\s*)',
+             bygroups(Text, Punctuation, Name, Text, Operator, Text,
+                      Keyword.Type, Text), 'defval'),
+            (r'\)', Operator, 'type')
+        ],
+        'type': [
+            (r'(\s*)(:)(\s*)(' + typeidentifier + r'|\*)',
+             bygroups(Text, Operator, Text, Keyword.Type), '#pop:2'),
+            (r'\s+', Text, '#pop:2'),
+            default('#pop:2')
+        ],
+        'defval': [
+            (r'(=)(\s*)([^(),]+)(\s*)(,?)',
+             bygroups(Operator, Text, using(this), Text, Operator), '#pop'),
+            (r',', Operator, '#pop'),
+            default('#pop')
+        ]
+    }
+
+    def analyse_text(text):
+        if re.match(r'\w+\s*:\s*\w', text):
+            return 0.3
+        return 0
+
+
+class MxmlLexer(RegexLexer):
+    """
+    For MXML markup.
+    Nested AS3 in <script> tags is highlighted by the appropriate lexer.
+
+    .. versionadded:: 1.1
+    """
+    flags = re.MULTILINE | re.DOTALL
+    name = 'MXML'
+    aliases = ['mxml']
+    filenames = ['*.mxml']
+    mimetimes = ['text/xml', 'application/xml']
+
+    tokens = {
+        'root': [
+            ('[^<&]+', Text),
+            (r'&\S*?;', Name.Entity),
+            (r'(\<\!\[CDATA\[)(.*?)(\]\]\>)',
+             bygroups(String, using(ActionScript3Lexer), String)),
+            ('<!--', Comment, 'comment'),
+            (r'<\?.*?\?>', Comment.Preproc),
+            ('<![^>]*>', Comment.Preproc),
+            (r'<\s*[\w:.-]+', Name.Tag, 'tag'),
+            (r'<\s*/\s*[\w:.-]+\s*>', Name.Tag),
+        ],
+        'comment': [
+            ('[^-]+', Comment),
+            ('-->', Comment, '#pop'),
+            ('-', Comment),
+        ],
+        'tag': [
+            (r'\s+', Text),
+            (r'[\w.:-]+\s*=', Name.Attribute, 'attr'),
+            (r'/?\s*>', Name.Tag, '#pop'),
+        ],
+        'attr': [
+            ('\s+', Text),
+            ('".*?"', String, '#pop'),
+            ("'.*?'", String, '#pop'),
+            (r'[^\s>]+', String, '#pop'),
+        ],
+    }

packages/wakatime/packages/pygments/lexers/agile.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.agile
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Just export lexer classes previously contained in this module.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexers.lisp import SchemeLexer
+from pygments.lexers.jvm import IokeLexer, ClojureLexer
+from pygments.lexers.python import PythonLexer, PythonConsoleLexer, \
+    PythonTracebackLexer, Python3Lexer, Python3TracebackLexer, DgLexer
+from pygments.lexers.ruby import RubyLexer, RubyConsoleLexer, FancyLexer
+from pygments.lexers.perl import PerlLexer, Perl6Lexer
+from pygments.lexers.d import CrocLexer, MiniDLexer
+from pygments.lexers.iolang import IoLexer
+from pygments.lexers.tcl import TclLexer
+from pygments.lexers.factor import FactorLexer
+from pygments.lexers.scripting import LuaLexer, MoonScriptLexer
+
+__all__ = []

packages/wakatime/packages/pygments/lexers/algebra.py

@@ -0,0 +1,221 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.algebra
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for computer algebra systems.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['GAPLexer', 'MathematicaLexer', 'MuPADLexer', 'BCLexer']
+
+
+class GAPLexer(RegexLexer):
+    """
+    For `GAP <http://www.gap-system.org>`_ source code.
+
+    .. versionadded:: 2.0
+    """
+    name = 'GAP'
+    aliases = ['gap']
+    filenames = ['*.g', '*.gd', '*.gi', '*.gap']
+
+    tokens = {
+        'root': [
+            (r'#.*$', Comment.Single),
+            (r'"(?:[^"\\]|\\.)*"', String),
+            (r'\(|\)|\[|\]|\{|\}', Punctuation),
+            (r'''(?x)\b(?:
+                if|then|elif|else|fi|
+                for|while|do|od|
+                repeat|until|
+                break|continue|
+                function|local|return|end|
+                rec|
+                quit|QUIT|
+                IsBound|Unbind|
+                TryNextMethod|
+                Info|Assert
+              )\b''', Keyword),
+            (r'''(?x)\b(?:
+                true|false|fail|infinity
+              )\b''',
+             Name.Constant),
+            (r'''(?x)\b(?:
+                (Declare|Install)([A-Z][A-Za-z]+)|
+                   BindGlobal|BIND_GLOBAL
+              )\b''',
+             Name.Builtin),
+            (r'\.|,|:=|;|=|\+|-|\*|/|\^|>|<', Operator),
+            (r'''(?x)\b(?:
+                and|or|not|mod|in
+              )\b''',
+             Operator.Word),
+            (r'''(?x)
+              (?:\w+|`[^`]*`)
+              (?:::\w+|`[^`]*`)*''', Name.Variable),
+            (r'[0-9]+(?:\.[0-9]*)?(?:e[0-9]+)?', Number),
+            (r'\.[0-9]+(?:e[0-9]+)?', Number),
+            (r'.', Text)
+        ],
+    }
+
+
+class MathematicaLexer(RegexLexer):
+    """
+    Lexer for `Mathematica <http://www.wolfram.com/mathematica/>`_ source code.
+
+    .. versionadded:: 2.0
+    """
+    name = 'Mathematica'
+    aliases = ['mathematica', 'mma', 'nb']
+    filenames = ['*.nb', '*.cdf', '*.nbp', '*.ma']
+    mimetypes = ['application/mathematica',
+                 'application/vnd.wolfram.mathematica',
+                 'application/vnd.wolfram.mathematica.package',
+                 'application/vnd.wolfram.cdf']
+
+    # http://reference.wolfram.com/mathematica/guide/Syntax.html
+    operators = (
+        ";;", "=", "=.", "!=" "==", ":=", "->", ":>", "/.", "+", "-", "*", "/",
+        "^", "&&", "||", "!", "<>", "|", "/;", "?", "@", "//", "/@", "@@",
+        "@@@", "~~", "===", "&", "<", ">", "<=", ">=",
+    )
+
+    punctuation = (",", ";", "(", ")", "[", "]", "{", "}")
+
+    def _multi_escape(entries):
+        return '(%s)' % ('|'.join(re.escape(entry) for entry in entries))
+
+    tokens = {
+        'root': [
+            (r'(?s)\(\*.*?\*\)', Comment),
+
+            (r'([a-zA-Z]+[A-Za-z0-9]*`)', Name.Namespace),
+            (r'([A-Za-z0-9]*_+[A-Za-z0-9]*)', Name.Variable),
+            (r'#\d*', Name.Variable),
+            (r'([a-zA-Z]+[a-zA-Z0-9]*)', Name),
+
+            (r'-?\d+\.\d*', Number.Float),
+            (r'-?\d*\.\d+', Number.Float),
+            (r'-?\d+', Number.Integer),
+
+            (words(operators), Operator),
+            (words(punctuation), Punctuation),
+            (r'".*?"', String),
+            (r'\s+', Text.Whitespace),
+        ],
+    }
+
+
+class MuPADLexer(RegexLexer):
+    """
+    A `MuPAD <http://www.mupad.com>`_ lexer.
+    Contributed by Christopher Creutzig <christopher@creutzig.de>.
+
+    .. versionadded:: 0.8
+    """
+    name = 'MuPAD'
+    aliases = ['mupad']
+    filenames = ['*.mu']
+
+    tokens = {
+        'root': [
+            (r'//.*?$', Comment.Single),
+            (r'/\*', Comment.Multiline, 'comment'),
+            (r'"(?:[^"\\]|\\.)*"', String),
+            (r'\(|\)|\[|\]|\{|\}', Punctuation),
+            (r'''(?x)\b(?:
+                next|break|end|
+                axiom|end_axiom|category|end_category|domain|end_domain|inherits|
+                if|%if|then|elif|else|end_if|
+                case|of|do|otherwise|end_case|
+                while|end_while|
+                repeat|until|end_repeat|
+                for|from|to|downto|step|end_for|
+                proc|local|option|save|begin|end_proc|
+                delete|frame
+              )\b''', Keyword),
+            (r'''(?x)\b(?:
+                DOM_ARRAY|DOM_BOOL|DOM_COMPLEX|DOM_DOMAIN|DOM_EXEC|DOM_EXPR|
+                DOM_FAIL|DOM_FLOAT|DOM_FRAME|DOM_FUNC_ENV|DOM_HFARRAY|DOM_IDENT|
+                DOM_INT|DOM_INTERVAL|DOM_LIST|DOM_NIL|DOM_NULL|DOM_POLY|DOM_PROC|
+                DOM_PROC_ENV|DOM_RAT|DOM_SET|DOM_STRING|DOM_TABLE|DOM_VAR
+              )\b''', Name.Class),
+            (r'''(?x)\b(?:
+                PI|EULER|E|CATALAN|
+                NIL|FAIL|undefined|infinity|
+                TRUE|FALSE|UNKNOWN
+              )\b''',
+             Name.Constant),
+            (r'\b(?:dom|procname)\b', Name.Builtin.Pseudo),
+            (r'\.|,|:|;|=|\+|-|\*|/|\^|@|>|<|\$|\||!|\'|%|~=', Operator),
+            (r'''(?x)\b(?:
+                and|or|not|xor|
+                assuming|
+                div|mod|
+                union|minus|intersect|in|subset
+              )\b''',
+             Operator.Word),
+            (r'\b(?:I|RDN_INF|RD_NINF|RD_NAN)\b', Number),
+            # (r'\b(?:adt|linalg|newDomain|hold)\b', Name.Builtin),
+            (r'''(?x)
+              ((?:[a-zA-Z_#][\w#]*|`[^`]*`)
+              (?:::[a-zA-Z_#][\w#]*|`[^`]*`)*)(\s*)([(])''',
+             bygroups(Name.Function, Text, Punctuation)),
+            (r'''(?x)
+              (?:[a-zA-Z_#][\w#]*|`[^`]*`)
+              (?:::[a-zA-Z_#][\w#]*|`[^`]*`)*''', Name.Variable),
+            (r'[0-9]+(?:\.[0-9]*)?(?:e[0-9]+)?', Number),
+            (r'\.[0-9]+(?:e[0-9]+)?', Number),
+            (r'.', Text)
+        ],
+        'comment': [
+            (r'[^*/]', Comment.Multiline),
+            (r'/\*', Comment.Multiline, '#push'),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'[*/]', Comment.Multiline)
+        ],
+    }
+
+
+class BCLexer(RegexLexer):
+    """
+    A `BC <https://www.gnu.org/software/bc/>`_ lexer.
+
+    .. versionadded:: 2.1
+    """
+    name = 'BC'
+    aliases = ['bc']
+    filenames = ['*.bc']
+
+    tokens = {
+        'root': [
+            (r'/\*', Comment.Multiline, 'comment'),
+            (r'"(?:[^"\\]|\\.)*"', String),
+            (r'[{}();,]', Punctuation),
+            (words(('if', 'else', 'while', 'for', 'break', 'continue',
+                    'halt', 'return', 'define', 'auto', 'print', 'read',
+                    'length', 'scale', 'sqrt', 'limits', 'quit',
+                    'warranty'), suffix=r'\b'), Keyword),
+            (r'\+\+|--|\|\||&&|'
+             r'([-<>+*%\^/!=])=?', Operator),
+            # bc doesn't support exponential
+            (r'[0-9]+(\.[0-9]*)?', Number),
+            (r'\.[0-9]+', Number),
+            (r'.', Text)
+        ],
+        'comment': [
+            (r'[^*/]+', Comment.Multiline),
+            (r'\*/', Comment.Multiline, '#pop'),
+            (r'[*/]', Comment.Multiline)
+        ],
+    }

packages/wakatime/packages/pygments/lexers/ambient.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.ambient
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for AmbientTalk language.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['AmbientTalkLexer']
+
+
+class AmbientTalkLexer(RegexLexer):
+    """
+    Lexer for `AmbientTalk <https://code.google.com/p/ambienttalk>`_ source code.
+
+    .. versionadded:: 2.0
+    """
+    name = 'AmbientTalk'
+    filenames = ['*.at']
+    aliases = ['at', 'ambienttalk', 'ambienttalk/2']
+    mimetypes = ['text/x-ambienttalk']
+
+    flags = re.MULTILINE | re.DOTALL
+
+    builtin = words(('if:', 'then:', 'else:', 'when:', 'whenever:', 'discovered:',
+                     'disconnected:', 'reconnected:', 'takenOffline:', 'becomes:',
+                     'export:', 'as:', 'object:', 'actor:', 'mirror:', 'taggedAs:',
+                     'mirroredBy:', 'is:'))
+    tokens = {
+        'root': [
+            (r'\s+', Text),
+            (r'//.*?\n', Comment.Single),
+            (r'/\*.*?\*/', Comment.Multiline),
+            (r'(def|deftype|import|alias|exclude)\b', Keyword),
+            (builtin, Name.Builtin),
+            (r'(true|false|nil)\b', Keyword.Constant),
+            (r'(~|lobby|jlobby|/)\.', Keyword.Constant, 'namespace'),
+            (r'"(\\\\|\\"|[^"])*"', String),
+            (r'\|', Punctuation, 'arglist'),
+            (r'<:|[*^!%&<>+=,./?-]|:=', Operator),
+            (r"`[a-zA-Z_]\w*", String.Symbol),
+            (r"[a-zA-Z_]\w*:", Name.Function),
+            (r"[{}()\[\];`]", Punctuation),
+            (r'(self|super)\b', Name.Variable.Instance),
+            (r"[a-zA-Z_]\w*", Name.Variable),
+            (r"@[a-zA-Z_]\w*", Name.Class),
+            (r"@\[", Name.Class, 'annotations'),
+            include('numbers'),
+        ],
+        'numbers': [
+            (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
+            (r'\d+', Number.Integer)
+        ],
+        'namespace': [
+            (r'[a-zA-Z_]\w*\.', Name.Namespace),
+            (r'[a-zA-Z_]\w*:', Name.Function, '#pop'),
+            (r'[a-zA-Z_]\w*(?!\.)', Name.Function, '#pop')
+        ],
+        'annotations': [
+            (r"(.*?)\]", Name.Class, '#pop')
+        ],
+        'arglist': [
+            (r'\|', Punctuation, '#pop'),
+            (r'\s*(,)\s*', Punctuation),
+            (r'[a-zA-Z_]\w*', Name.Variable),
+        ],
+    }

packages/wakatime/packages/pygments/lexers/ampl.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.ampl
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the ampl language. <http://ampl.com/>
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, using, this, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['AmplLexer']
+
+
+class AmplLexer(RegexLexer):
+    """
+    For AMPL source code.
+
+    .. versionadded:: 2.2
+    """
+    name = 'Ampl'
+    aliases = ['ampl']
+    filenames = ['*.run']
+
+    tokens = {
+        'root': [
+            (r'\n', Text),
+            (r'\s+', Text.Whitespace),
+            (r'#.*?\n', Comment.Single),
+            (r'/[*](.|\n)*?[*]/', Comment.Multiline),
+            (words((
+                'call', 'cd', 'close', 'commands', 'data', 'delete', 'display',
+                'drop', 'end', 'environ', 'exit', 'expand', 'include', 'load',
+                'model', 'objective', 'option', 'problem', 'purge', 'quit',
+                'redeclare', 'reload', 'remove', 'reset', 'restore', 'shell',
+                'show', 'solexpand', 'solution', 'solve', 'update', 'unload',
+                'xref', 'coeff', 'coef', 'cover', 'obj', 'interval', 'default',
+                'from', 'to', 'to_come', 'net_in', 'net_out', 'dimen',
+                'dimension', 'check', 'complements', 'write', 'function',
+                'pipe', 'format', 'if', 'then', 'else', 'in', 'while', 'repeat',
+                'for'), suffix=r'\b'), Keyword.Reserved),
+            (r'(integer|binary|symbolic|ordered|circular|reversed|INOUT|IN|OUT|LOCAL)',
+             Keyword.Type),
+            (r'\".*?\"', String.Double),
+            (r'\'.*?\'', String.Single),
+            (r'[()\[\]{},;:]+', Punctuation),
+            (r'\b(\w+)(\.)(astatus|init0|init|lb0|lb1|lb2|lb|lrc|'
+             r'lslack|rc|relax|slack|sstatus|status|ub0|ub1|ub2|'
+             r'ub|urc|uslack|val)',
+             bygroups(Name.Variable, Punctuation, Keyword.Reserved)),
+            (r'(set|param|var|arc|minimize|maximize|subject to|s\.t\.|subj to|'
+             r'node|table|suffix|read table|write table)(\s+)(\w+)',
+             bygroups(Keyword.Declaration, Text, Name.Variable)),
+            (r'(param)(\s*)(:)(\s*)(\w+)(\s*)(:)(\s*)((\w|\s)+)',
+             bygroups(Keyword.Declaration, Text, Punctuation, Text,
+                      Name.Variable, Text, Punctuation, Text, Name.Variable)),
+            (r'(let|fix|unfix)(\s*)((?:\{.*\})?)(\s*)(\w+)',
+             bygroups(Keyword.Declaration, Text, using(this), Text, Name.Variable)),
+            (words((
+                'abs', 'acos', 'acosh', 'alias', 'asin', 'asinh', 'atan', 'atan2',
+                'atanh', 'ceil', 'ctime', 'cos', 'exp', 'floor', 'log', 'log10',
+                'max', 'min', 'precision', 'round', 'sin', 'sinh', 'sqrt', 'tan',
+                'tanh', 'time', 'trunc', 'Beta', 'Cauchy', 'Exponential', 'Gamma',
+                'Irand224', 'Normal', 'Normal01', 'Poisson', 'Uniform', 'Uniform01',
+                'num', 'num0', 'ichar', 'char', 'length', 'substr', 'sprintf',
+                'match', 'sub', 'gsub', 'print', 'printf', 'next', 'nextw', 'prev',
+                'prevw', 'first', 'last', 'ord', 'ord0', 'card', 'arity',
+                'indexarity'), prefix=r'\b', suffix=r'\b'), Name.Builtin),
+            (r'(\+|\-|\*|/|\*\*|=|<=|>=|==|\||\^|<|>|\!|\.\.|:=|\&|\!=|<<|>>)',
+             Operator),
+            (words((
+                'or', 'exists', 'forall', 'and', 'in', 'not', 'within', 'union',
+                'diff', 'difference', 'symdiff', 'inter', 'intersect',
+                'intersection', 'cross', 'setof', 'by', 'less', 'sum', 'prod',
+                'product', 'div', 'mod'), suffix=r'\b'),
+             Keyword.Reserved),  # Operator.Name but not enough emphasized with that
+            (r'(\d+\.(?!\.)\d*|\.(?!.)\d+)([eE][+-]?\d+)?', Number.Float),
+            (r'\d+([eE][+-]?\d+)?', Number.Integer),
+            (r'[+-]?Infinity', Number.Integer),
+            (r'(\w+|(\.(?!\.)))', Text)
+        ]
+
+    }

packages/wakatime/packages/pygments/lexers/apl.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.apl
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for APL.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation
+
+__all__ = ['APLLexer']
+
+
+class APLLexer(RegexLexer):
+    """
+    A simple APL lexer.
+
+    .. versionadded:: 2.0
+    """
+    name = 'APL'
+    aliases = ['apl']
+    filenames = ['*.apl']
+
+    tokens = {
+        'root': [
+            # Whitespace
+            # ==========
+            (r'\s+', Text),
+            #
+            # Comment
+            # =======
+            # '⍝' is traditional; '#' is supported by GNU APL and NGN (but not Dyalog)
+            (u'[⍝#].*$', Comment.Single),
+            #
+            # Strings
+            # =======
+            (r'\'((\'\')|[^\'])*\'', String.Single),
+            (r'"(("")|[^"])*"', String.Double),  # supported by NGN APL
+            #
+            # Punctuation
+            # ===========
+            # This token type is used for diamond and parenthesis
+            # but not for bracket and ; (see below)
+            (u'[⋄◇()]', Punctuation),
+            #
+            # Array indexing
+            # ==============
+            # Since this token type is very important in APL, it is not included in
+            # the punctuation token type but rather in the following one
+            (r'[\[\];]', String.Regex),
+            #
+            # Distinguished names
+            # ===================
+            # following IBM APL2 standard
+            (u'⎕[A-Za-zΔ∆⍙][A-Za-zΔ∆⍙_¯0-9]*', Name.Function),
+            #
+            # Labels
+            # ======
+            # following IBM APL2 standard
+            # (u'[A-Za-zΔ∆⍙][A-Za-zΔ∆⍙_¯0-9]*:', Name.Label),
+            #
+            # Variables
+            # =========
+            # following IBM APL2 standard
+            (u'[A-Za-zΔ∆⍙][A-Za-zΔ∆⍙_¯0-9]*', Name.Variable),
+            #
+            # Numbers
+            # =======
+            (u'¯?(0[Xx][0-9A-Fa-f]+|[0-9]*\.?[0-9]+([Ee][+¯]?[0-9]+)?|¯|∞)'
+             u'([Jj]¯?(0[Xx][0-9A-Fa-f]+|[0-9]*\.?[0-9]+([Ee][+¯]?[0-9]+)?|¯|∞))?',
+             Number),
+            #
+            # Operators
+            # ==========
+            (u'[\.\\\/⌿⍀¨⍣⍨⍠⍤∘]', Name.Attribute),  # closest token type
+            (u'[+\-×÷⌈⌊∣|⍳?*⍟○!⌹<≤=>≥≠≡≢∊⍷∪∩~∨∧⍱⍲⍴,⍪⌽⊖⍉↑↓⊂⊃⌷⍋⍒⊤⊥⍕⍎⊣⊢⍁⍂≈⌸⍯↗]',
+             Operator),
+            #
+            # Constant
+            # ========
+            (u'⍬', Name.Constant),
+            #
+            # Quad symbol
+            # ===========
+            (u'[⎕⍞]', Name.Variable.Global),
+            #
+            # Arrows left/right
+            # =================
+            (u'[←→]', Keyword.Declaration),
+            #
+            # D-Fn
+            # ====
+            (u'[⍺⍵⍶⍹∇:]', Name.Builtin.Pseudo),
+            (r'[{}]', Keyword.Type),
+        ],
+    }

packages/wakatime/packages/pygments/lexers/archetype.py

@@ -0,0 +1,318 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.archetype
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Archetype-related syntaxes, including:
+
+    - ODIN syntax <https://github.com/openEHR/odin>
+    - ADL syntax <http://www.openehr.org/releases/trunk/architecture/am/adl2.pdf>
+    - cADL sub-syntax of ADL
+
+    For uses of this syntax, see the openEHR archetypes <http://www.openEHR.org/ckm>
+
+    Contributed by Thomas Beale <https://github.com/wolandscat>,
+    <https://bitbucket.org/thomas_beale>.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, bygroups, using, default
+from pygments.token import Text, Comment, Name, Literal, Number, String, \
+    Punctuation, Keyword, Operator, Generic
+
+__all__ = ['OdinLexer', 'CadlLexer', 'AdlLexer']
+
+
+class AtomsLexer(RegexLexer):
+    """
+    Lexer for Values used in ADL and ODIN.
+
+    .. versionadded:: 2.1
+    """
+
+    tokens = {
+        # ----- pseudo-states for inclusion -----
+        'whitespace': [
+            (r'\n', Text),
+            (r'\s+', Text),
+            (r'[ \t]*--.*$', Comment),
+        ],
+        'archetype_id': [
+            (r'[ \t]*([a-zA-Z]\w+(\.[a-zA-Z]\w+)*::)?[a-zA-Z]\w+(-[a-zA-Z]\w+){2}'
+             r'\.\w+[\w-]*\.v\d+(\.\d+){,2}((-[a-z]+)(\.\d+)?)?', Name.Decorator),
+        ],
+        'date_constraints': [
+            # ISO 8601-based date/time constraints
+            (r'[Xx?YyMmDdHhSs\d]{2,4}([:-][Xx?YyMmDdHhSs\d]{2}){2}', Literal.Date),
+            # ISO 8601-based duration constraints + optional trailing slash
+            (r'(P[YyMmWwDd]+(T[HhMmSs]+)?|PT[HhMmSs]+)/?', Literal.Date),
+        ],
+        'ordered_values': [
+            # ISO 8601 date with optional 'T' ligature
+            (r'\d{4}-\d{2}-\d{2}T?', Literal.Date),
+            # ISO 8601 time
+            (r'\d{2}:\d{2}:\d{2}(\.\d+)?([+-]\d{4}|Z)?', Literal.Date),
+            # ISO 8601 duration
+            (r'P((\d*(\.\d+)?[YyMmWwDd]){1,3}(T(\d*(\.\d+)?[HhMmSs]){,3})?|'
+             r'T(\d*(\.\d+)?[HhMmSs]){,3})', Literal.Date),
+            (r'[+-]?(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+', Number.Float),
+            (r'[+-]?(\d+)*\.\d+%?', Number.Float),
+            (r'0x[0-9a-fA-F]+', Number.Hex),
+            (r'[+-]?\d+%?', Number.Integer),
+        ],
+        'values': [
+            include('ordered_values'),
+            (r'([Tt]rue|[Ff]alse)', Literal),
+            (r'"', String, 'string'),
+            (r"'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char),
+            (r'[a-z][a-z0-9+.-]*:', Literal, 'uri'),
+            # term code
+            (r'(\[)(\w[\w-]*(?:\([^)\n]+\))?)(::)(\w[\w-]*)(\])',
+             bygroups(Punctuation, Name.Decorator, Punctuation, Name.Decorator,
+                      Punctuation)),
+            (r'\|', Punctuation, 'interval'),
+            # list continuation
+            (r'\.\.\.', Punctuation),
+        ],
+        'constraint_values': [
+            (r'(\[)(\w[\w-]*(?:\([^)\n]+\))?)(::)',
+             bygroups(Punctuation, Name.Decorator, Punctuation), 'adl14_code_constraint'),
+            # ADL 1.4 ordinal constraint
+            (r'(\d*)(\|)(\[\w[\w-]*::\w[\w-]*\])((?:[,;])?)',
+             bygroups(Number, Punctuation, Name.Decorator, Punctuation)),
+            include('date_constraints'),
+            include('values'),
+        ],
+
+        # ----- real states -----
+        'string': [
+            ('"', String, '#pop'),
+            (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|'
+             r'u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|[0-7]{1,3})', String.Escape),
+            # all other characters
+            (r'[^\\"]+', String),
+            # stray backslash
+            (r'\\', String),
+        ],
+        'uri': [
+            # effective URI terminators
+            (r'[,>\s]', Punctuation, '#pop'),
+            (r'[^>\s,]+', Literal),
+        ],
+        'interval': [
+            (r'\|', Punctuation, '#pop'),
+            include('ordered_values'),
+            (r'\.\.', Punctuation),
+            (r'[<>=] *', Punctuation),
+            # handle +/-
+            (r'\+/-', Punctuation),
+            (r'\s+', Text),
+        ],
+        'any_code': [
+            include('archetype_id'),
+            # if it is a code
+            (r'[a-z_]\w*[0-9.]+(@[^\]]+)?', Name.Decorator),
+            # if it is tuple with attribute names
+            (r'[a-z_]\w*', Name.Class),
+            # if it is an integer, i.e. Xpath child index
+            (r'[0-9]+', Text),
+            (r'\|', Punctuation, 'code_rubric'),
+            (r'\]', Punctuation, '#pop'),
+            # handle use_archetype statement
+            (r'\s*,\s*', Punctuation),
+        ],
+        'code_rubric': [
+            (r'\|', Punctuation, '#pop'),
+            (r'[^|]+', String),
+        ],
+        'adl14_code_constraint': [
+            (r'\]', Punctuation, '#pop'),
+            (r'\|', Punctuation, 'code_rubric'),
+            (r'(\w[\w-]*)([;,]?)', bygroups(Name.Decorator, Punctuation)),
+            include('whitespace'),
+        ],
+    }
+
+
+class OdinLexer(AtomsLexer):
+    """
+    Lexer for ODIN syntax.
+
+    .. versionadded:: 2.1
+    """
+    name = 'ODIN'
+    aliases = ['odin']
+    filenames = ['*.odin']
+    mimetypes = ['text/odin']
+
+    tokens = {
+        'path': [
+            (r'>', Punctuation, '#pop'),
+            # attribute name
+            (r'[a-z_]\w*', Name.Class),
+            (r'/', Punctuation),
+            (r'\[', Punctuation, 'key'),
+            (r'\s*,\s*', Punctuation, '#pop'),
+            (r'\s+', Text, '#pop'),
+        ],
+        'key': [
+            include('values'),
+            (r'\]', Punctuation, '#pop'),
+        ],
+        'type_cast': [
+            (r'\)', Punctuation, '#pop'),
+            (r'[^)]+',  Name.Class),
+        ],
+        'root': [
+            include('whitespace'),
+            (r'([Tt]rue|[Ff]alse)', Literal),
+            include('values'),
+            # x-ref path
+            (r'/', Punctuation, 'path'),
+            # x-ref path starting with key
+            (r'\[', Punctuation, 'key'),
+            # attribute name
+            (r'[a-z_]\w*', Name.Class),
+            (r'=', Operator),
+            (r'\(', Punctuation, 'type_cast'),
+            (r',', Punctuation),
+            (r'<', Punctuation),
+            (r'>', Punctuation),
+            (r';', Punctuation),
+        ],
+    }
+
+
+class CadlLexer(AtomsLexer):
+    """
+    Lexer for cADL syntax.
+
+    .. versionadded:: 2.1
+    """
+    name = 'cADL'
+    aliases = ['cadl']
+    filenames = ['*.cadl']
+
+    tokens = {
+        'path': [
+            # attribute name
+            (r'[a-z_]\w*', Name.Class),
+            (r'/', Punctuation),
+            (r'\[', Punctuation, 'any_code'),
+            (r'\s+', Punctuation, '#pop'),
+        ],
+        'root': [
+            include('whitespace'),
+            (r'(cardinality|existence|occurrences|group|include|exclude|'
+             r'allow_archetype|use_archetype|use_node)\W', Keyword.Type),
+            (r'(and|or|not|there_exists|xor|implies|for_all)\W', Keyword.Type),
+            (r'(after|before|closed)\W', Keyword.Type),
+            (r'(not)\W', Operator),
+            (r'(matches|is_in)\W', Operator),
+            # is_in / not is_in char
+            (u'(\u2208|\u2209)', Operator),
+            # there_exists / not there_exists / for_all / and / or
+            (u'(\u2203|\u2204|\u2200|\u2227|\u2228|\u22BB|\223C)',
+             Operator),
+            # regex in slot or as string constraint
+            (r'(\{)(\s*/[^}]+/\s*)(\})',
+             bygroups(Punctuation, String.Regex, Punctuation)),
+            # regex in slot or as string constraint
+            (r'(\{)(\s*\^[^}]+\^\s*)(\})',
+             bygroups(Punctuation, String.Regex, Punctuation)),
+            (r'/', Punctuation, 'path'),
+            # for cardinality etc
+            (r'(\{)((?:\d+\.\.)?(?:\d+|\*))'
+             r'((?:\s*;\s*(?:ordered|unordered|unique)){,2})(\})',
+             bygroups(Punctuation, Number, Number, Punctuation)),
+            # [{ is start of a tuple value
+            (r'\[\{', Punctuation),
+            (r'\}\]', Punctuation),
+            (r'\{', Punctuation),
+            (r'\}', Punctuation),
+            include('constraint_values'),
+            # type name
+            (r'[A-Z]\w+(<[A-Z]\w+([A-Za-z_<>]*)>)?',  Name.Class),
+            # attribute name
+            (r'[a-z_]\w*', Name.Class),
+            (r'\[', Punctuation, 'any_code'),
+            (r'(~|//|\\\\|\+|-|/|\*|\^|!=|=|<=|>=|<|>]?)', Operator),
+            (r'\(', Punctuation),
+            (r'\)', Punctuation),
+            # for lists of values
+            (r',', Punctuation),
+            (r'"', String, 'string'),
+            # for assumed value
+            (r';', Punctuation),
+        ],
+    }
+
+
+class AdlLexer(AtomsLexer):
+    """
+    Lexer for ADL syntax.
+
+    .. versionadded:: 2.1
+    """
+
+    name = 'ADL'
+    aliases = ['adl']
+    filenames = ['*.adl', '*.adls', '*.adlf', '*.adlx']
+
+    tokens = {
+        'whitespace': [
+            # blank line ends
+            (r'\s*\n', Text),
+            # comment-only line
+            (r'^[ \t]*--.*$', Comment),
+        ],
+        'odin_section': [
+            # repeating the following two rules from the root state enable multi-line
+            # strings that start in the first column to be dealt with
+            (r'^(language|description|ontology|terminology|annotations|'
+             r'component_terminologies|revision_history)[ \t]*\n', Generic.Heading),
+            (r'^(definition)[ \t]*\n', Generic.Heading, 'cadl_section'),
+            (r'^([ \t]*|[ \t]+.*)\n', using(OdinLexer)),
+            (r'^([^"]*")(>[ \t]*\n)', bygroups(String, Punctuation)),
+            # template overlay delimiter
+            (r'^----------*\n', Text, '#pop'),
+            (r'^.*\n', String),
+            default('#pop'),
+        ],
+        'cadl_section': [
+            (r'^([ \t]*|[ \t]+.*)\n', using(CadlLexer)),
+            default('#pop'),
+        ],
+        'rules_section': [
+            (r'^[ \t]+.*\n', using(CadlLexer)),
+            default('#pop'),
+        ],
+        'metadata': [
+            (r'\)', Punctuation, '#pop'),
+            (r';', Punctuation),
+            (r'([Tt]rue|[Ff]alse)', Literal),
+            # numbers and version ids
+            (r'\d+(\.\d+)*', Literal),
+            # Guids
+            (r'(\d|[a-fA-F])+(-(\d|[a-fA-F])+){3,}', Literal),
+            (r'\w+', Name.Class),
+            (r'"', String, 'string'),
+            (r'=', Operator),
+            (r'[ \t]+', Text),
+            default('#pop'),
+        ],
+        'root': [
+            (r'^(archetype|template_overlay|operational_template|template|'
+             r'speciali[sz]e)', Generic.Heading),
+            (r'^(language|description|ontology|terminology|annotations|'
+             r'component_terminologies|revision_history)[ \t]*\n',
+             Generic.Heading, 'odin_section'),
+            (r'^(definition)[ \t]*\n', Generic.Heading, 'cadl_section'),
+            (r'^(rules)[ \t]*\n', Generic.Heading, 'rules_section'),
+            include('archetype_id'),
+            (r'[ \t]*\(', Punctuation, 'metadata'),
+            include('whitespace'),
+        ],
+    }

packages/wakatime/packages/pygments/lexers/asm.py

@@ -0,0 +1,641 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.asm
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexers for assembly languages.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using, words, \
+    DelegatingLexer
+from pygments.lexers.c_cpp import CppLexer, CLexer
+from pygments.lexers.d import DLexer
+from pygments.token import Text, Name, Number, String, Comment, Punctuation, \
+    Other, Keyword, Operator
+
+__all__ = ['GasLexer', 'ObjdumpLexer', 'DObjdumpLexer', 'CppObjdumpLexer',
+           'CObjdumpLexer', 'HsailLexer', 'LlvmLexer', 'NasmLexer',
+           'NasmObjdumpLexer', 'TasmLexer', 'Ca65Lexer']
+
+
+class GasLexer(RegexLexer):
+    """
+    For Gas (AT&T) assembly code.
+    """
+    name = 'GAS'
+    aliases = ['gas', 'asm']
+    filenames = ['*.s', '*.S']
+    mimetypes = ['text/x-gas']
+
+    #: optional Comment or Whitespace
+    string = r'"(\\"|[^"])*"'
+    char = r'[\w$.@-]'
+    identifier = r'(?:[a-zA-Z$_]' + char + '*|\.' + char + '+)'
+    number = r'(?:0[xX][a-zA-Z0-9]+|\d+)'
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            (identifier + ':', Name.Label),
+            (r'\.' + identifier, Name.Attribute, 'directive-args'),
+            (r'lock|rep(n?z)?|data\d+', Name.Attribute),
+            (identifier, Name.Function, 'instruction-args'),
+            (r'[\r\n]+', Text)
+        ],
+        'directive-args': [
+            (identifier, Name.Constant),
+            (string, String),
+            ('@' + identifier, Name.Attribute),
+            (number, Number.Integer),
+            (r'[\r\n]+', Text, '#pop'),
+
+            include('punctuation'),
+            include('whitespace')
+        ],
+        'instruction-args': [
+            # For objdump-disassembled code, shouldn't occur in
+            # actual assembler input
+            ('([a-z0-9]+)( )(<)('+identifier+')(>)',
+                bygroups(Number.Hex, Text, Punctuation, Name.Constant,
+                         Punctuation)),
+            ('([a-z0-9]+)( )(<)('+identifier+')([-+])('+number+')(>)',
+                bygroups(Number.Hex, Text, Punctuation, Name.Constant,
+                         Punctuation, Number.Integer, Punctuation)),
+
+            # Address constants
+            (identifier, Name.Constant),
+            (number, Number.Integer),
+            # Registers
+            ('%' + identifier, Name.Variable),
+            # Numeric constants
+            ('$'+number, Number.Integer),
+            (r"$'(.|\\')'", String.Char),
+            (r'[\r\n]+', Text, '#pop'),
+
+            include('punctuation'),
+            include('whitespace')
+        ],
+        'whitespace': [
+            (r'\n', Text),
+            (r'\s+', Text),
+            (r'[;#].*?\n', Comment)
+        ],
+        'punctuation': [
+            (r'[-*,.()\[\]!:]+', Punctuation)
+        ]
+    }
+
+    def analyse_text(text):
+        if re.match(r'^\.(text|data|section)', text, re.M):
+            return True
+        elif re.match(r'^\.\w+', text, re.M):
+            return 0.1
+
+
+def _objdump_lexer_tokens(asm_lexer):
+    """
+    Common objdump lexer tokens to wrap an ASM lexer.
+    """
+    hex_re = r'[0-9A-Za-z]'
+    return {
+        'root': [
+            # File name & format:
+            ('(.*?)(:)( +file format )(.*?)$',
+                bygroups(Name.Label, Punctuation, Text, String)),
+            # Section header
+            ('(Disassembly of section )(.*?)(:)$',
+                bygroups(Text, Name.Label, Punctuation)),
+            # Function labels
+            # (With offset)
+            ('('+hex_re+'+)( )(<)(.*?)([-+])(0[xX][A-Za-z0-9]+)(>:)$',
+                bygroups(Number.Hex, Text, Punctuation, Name.Function,
+                         Punctuation, Number.Hex, Punctuation)),
+            # (Without offset)
+            ('('+hex_re+'+)( )(<)(.*?)(>:)$',
+                bygroups(Number.Hex, Text, Punctuation, Name.Function,
+                         Punctuation)),
+            # Code line with disassembled instructions
+            ('( *)('+hex_re+r'+:)(\t)((?:'+hex_re+hex_re+' )+)( *\t)([a-zA-Z].*?)$',
+                bygroups(Text, Name.Label, Text, Number.Hex, Text,
+                         using(asm_lexer))),
+            # Code line with ascii
+            ('( *)('+hex_re+r'+:)(\t)((?:'+hex_re+hex_re+' )+)( *)(.*?)$',
+                bygroups(Text, Name.Label, Text, Number.Hex, Text, String)),
+            # Continued code line, only raw opcodes without disassembled
+            # instruction
+            ('( *)('+hex_re+r'+:)(\t)((?:'+hex_re+hex_re+' )+)$',
+                bygroups(Text, Name.Label, Text, Number.Hex)),
+            # Skipped a few bytes
+            (r'\t\.\.\.$', Text),
+            # Relocation line
+            # (With offset)
+            (r'(\t\t\t)('+hex_re+r'+:)( )([^\t]+)(\t)(.*?)([-+])(0x'+hex_re+'+)$',
+                bygroups(Text, Name.Label, Text, Name.Property, Text,
+                         Name.Constant, Punctuation, Number.Hex)),
+            # (Without offset)
+            (r'(\t\t\t)('+hex_re+r'+:)( )([^\t]+)(\t)(.*?)$',
+                bygroups(Text, Name.Label, Text, Name.Property, Text,
+                         Name.Constant)),
+            (r'[^\n]+\n', Other)
+        ]
+    }
+
+
+class ObjdumpLexer(RegexLexer):
+    """
+    For the output of 'objdump -dr'
+    """
+    name = 'objdump'
+    aliases = ['objdump']
+    filenames = ['*.objdump']
+    mimetypes = ['text/x-objdump']
+
+    tokens = _objdump_lexer_tokens(GasLexer)
+
+
+class DObjdumpLexer(DelegatingLexer):
+    """
+    For the output of 'objdump -Sr on compiled D files'
+    """
+    name = 'd-objdump'
+    aliases = ['d-objdump']
+    filenames = ['*.d-objdump']
+    mimetypes = ['text/x-d-objdump']
+
+    def __init__(self, **options):
+        super(DObjdumpLexer, self).__init__(DLexer, ObjdumpLexer, **options)
+
+
+class CppObjdumpLexer(DelegatingLexer):
+    """
+    For the output of 'objdump -Sr on compiled C++ files'
+    """
+    name = 'cpp-objdump'
+    aliases = ['cpp-objdump', 'c++-objdumb', 'cxx-objdump']
+    filenames = ['*.cpp-objdump', '*.c++-objdump', '*.cxx-objdump']
+    mimetypes = ['text/x-cpp-objdump']
+
+    def __init__(self, **options):
+        super(CppObjdumpLexer, self).__init__(CppLexer, ObjdumpLexer, **options)
+
+
+class CObjdumpLexer(DelegatingLexer):
+    """
+    For the output of 'objdump -Sr on compiled C files'
+    """
+    name = 'c-objdump'
+    aliases = ['c-objdump']
+    filenames = ['*.c-objdump']
+    mimetypes = ['text/x-c-objdump']
+
+    def __init__(self, **options):
+        super(CObjdumpLexer, self).__init__(CLexer, ObjdumpLexer, **options)
+
+
+class HsailLexer(RegexLexer):
+    """
+    For HSAIL assembly code.
+
+    .. versionadded:: 2.2
+    """
+    name = 'HSAIL'
+    aliases = ['hsail', 'hsa']
+    filenames = ['*.hsail']
+    mimetypes = ['text/x-hsail']
+
+    string = r'"[^"]*?"'
+    identifier = r'[a-zA-Z_][\w.]*'
+    # Registers
+    register_number = r'[0-9]+'
+    register = r'(\$(c|s|d|q)' + register_number + ')'
+    # Qualifiers
+    alignQual = r'(align\(\d+\))'
+    widthQual = r'(width\((\d+|all)\))'
+    allocQual = r'(alloc\(agent\))'
+    # Instruction Modifiers
+    roundingMod = (r'((_ftz)?(_up|_down|_zero|_near))')
+    datatypeMod = (r'_('
+                   # packedTypes
+                   r'u8x4|s8x4|u16x2|s16x2|u8x8|s8x8|u16x4|s16x4|u32x2|s32x2|'
+                   r'u8x16|s8x16|u16x8|s16x8|u32x4|s32x4|u64x2|s64x2|'
+                   r'f16x2|f16x4|f16x8|f32x2|f32x4|f64x2|'
+                   # baseTypes
+                   r'u8|s8|u16|s16|u32|s32|u64|s64|'
+                   r'b128|b8|b16|b32|b64|b1|'
+                   r'f16|f32|f64|'
+                   # opaqueType
+                   r'roimg|woimg|rwimg|samp|sig32|sig64)')
+
+    # Numeric Constant
+    float = r'((\d+\.)|(\d*\.\d+))[eE][+-]?\d+'
+    hexfloat = r'0[xX](([0-9a-fA-F]+\.[0-9a-fA-F]*)|([0-9a-fA-F]*\.[0-9a-fA-F]+))[pP][+-]?\d+'
+    ieeefloat = r'0((h|H)[0-9a-fA-F]{4}|(f|F)[0-9a-fA-F]{8}|(d|D)[0-9a-fA-F]{16})'
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+            include('comments'),
+
+            (string, String),
+
+            (r'@' + identifier + ':?', Name.Label),
+
+            (register, Name.Variable.Anonymous),
+
+            include('keyword'),
+
+            (r'&' + identifier, Name.Variable.Global),
+            (r'%' + identifier, Name.Variable),
+
+            (hexfloat, Number.Hex),
+            (r'0[xX][a-fA-F0-9]+', Number.Hex),
+            (ieeefloat, Number.Float),
+            (float, Number.Float),
+            ('\d+', Number.Integer),
+
+            (r'[=<>{}\[\]()*.,:;!]|x\b', Punctuation)
+        ],
+        'whitespace': [
+            (r'(\n|\s)+', Text),
+        ],
+        'comments': [
+            (r'/\*.*?\*/', Comment.Multiline),
+            (r'//.*?\n', Comment.Single),
+        ],
+        'keyword': [
+            # Types
+            (r'kernarg' + datatypeMod, Keyword.Type),
+
+            # Regular keywords
+            (r'\$(full|base|small|large|default|zero|near)', Keyword),
+            (words((
+                'module', 'extension', 'pragma', 'prog', 'indirect', 'signature',
+                'decl', 'kernel', 'function', 'enablebreakexceptions',
+                'enabledetectexceptions', 'maxdynamicgroupsize', 'maxflatgridsize',
+                'maxflatworkgroupsize', 'requireddim', 'requiredgridsize',
+                'requiredworkgroupsize', 'requirenopartialworkgroups'),
+                suffix=r'\b'), Keyword),
+
+            # instructions
+            (roundingMod, Keyword),
+            (datatypeMod, Keyword),
+            (r'_(' + alignQual + '|' + widthQual + ')', Keyword),
+            (r'_kernarg', Keyword),
+            (r'(nop|imagefence)\b', Keyword),
+            (words((
+                'cleardetectexcept', 'clock', 'cuid', 'debugtrap', 'dim',
+                'getdetectexcept', 'groupbaseptr', 'kernargbaseptr', 'laneid',
+                'maxcuid', 'maxwaveid', 'packetid', 'setdetectexcept', 'waveid',
+                'workitemflatabsid', 'workitemflatid', 'nullptr', 'abs', 'bitrev',
+                'currentworkgroupsize', 'currentworkitemflatid', 'fract', 'ncos',
+                'neg', 'nexp2', 'nlog2', 'nrcp', 'nrsqrt', 'nsin', 'nsqrt',
+                'gridgroups', 'gridsize', 'not', 'sqrt', 'workgroupid',
+                'workgroupsize', 'workitemabsid', 'workitemid', 'ceil', 'floor',
+                'rint', 'trunc', 'add', 'bitmask', 'borrow', 'carry', 'copysign',
+                'div', 'rem', 'sub', 'shl', 'shr', 'and', 'or', 'xor', 'unpackhi',
+                'unpacklo', 'max', 'min', 'fma', 'mad', 'bitextract', 'bitselect',
+                'shuffle', 'cmov', 'bitalign', 'bytealign', 'lerp', 'nfma', 'mul',
+                'mulhi', 'mul24hi', 'mul24', 'mad24', 'mad24hi', 'bitinsert',
+                'combine', 'expand', 'lda', 'mov', 'pack', 'unpack', 'packcvt',
+                'unpackcvt', 'sad', 'sementp', 'ftos', 'stof', 'cmp', 'ld', 'st',
+                '_eq', '_ne', '_lt', '_le', '_gt', '_ge', '_equ', '_neu', '_ltu',
+                '_leu', '_gtu', '_geu', '_num', '_nan', '_seq', '_sne', '_slt',
+                '_sle', '_sgt', '_sge', '_snum', '_snan', '_sequ', '_sneu', '_sltu',
+                '_sleu', '_sgtu', '_sgeu', 'atomic', '_ld', '_st', '_cas', '_add',
+                '_and', '_exch', '_max', '_min', '_or', '_sub', '_wrapdec',
+                '_wrapinc', '_xor', 'ret', 'cvt', '_readonly', '_kernarg', '_global',
+                'br', 'cbr', 'sbr', '_scacq', '_screl', '_scar', '_rlx', '_wave',
+                '_wg', '_agent', '_system', 'ldimage', 'stimage', '_v2', '_v3', '_v4',
+                '_1d', '_2d', '_3d', '_1da', '_2da', '_1db', '_2ddepth', '_2dadepth',
+                '_width', '_height', '_depth', '_array', '_channelorder',
+                '_channeltype', 'querysampler', '_coord', '_filter', '_addressing',
+                'barrier', 'wavebarrier', 'initfbar', 'joinfbar', 'waitfbar',
+                'arrivefbar', 'leavefbar', 'releasefbar', 'ldf', 'activelaneid',
+                'activelanecount', 'activelanemask', 'activelanepermute', 'call',
+                'scall', 'icall', 'alloca', 'packetcompletionsig',
+                'addqueuewriteindex', 'casqueuewriteindex', 'ldqueuereadindex',
+                'stqueuereadindex', 'readonly', 'global', 'private', 'group',
+                'spill', 'arg', '_upi', '_downi', '_zeroi', '_neari', '_upi_sat',
+                '_downi_sat', '_zeroi_sat', '_neari_sat', '_supi', '_sdowni',
+                '_szeroi', '_sneari', '_supi_sat', '_sdowni_sat', '_szeroi_sat',
+                '_sneari_sat', '_pp', '_ps', '_sp', '_ss', '_s', '_p', '_pp_sat',
+                '_ps_sat', '_sp_sat', '_ss_sat', '_s_sat', '_p_sat')), Keyword),
+
+            # Integer types
+            (r'i[1-9]\d*', Keyword)
+        ]
+    }
+
+
+class LlvmLexer(RegexLexer):
+    """
+    For LLVM assembly code.
+    """
+    name = 'LLVM'
+    aliases = ['llvm']
+    filenames = ['*.ll']
+    mimetypes = ['text/x-llvm']
+
+    #: optional Comment or Whitespace
+    string = r'"[^"]*?"'
+    identifier = r'([-a-zA-Z$._][\w\-$.]*|' + string + ')'
+
+    tokens = {
+        'root': [
+            include('whitespace'),
+
+            # Before keywords, because keywords are valid label names :(...
+            (identifier + '\s*:', Name.Label),
+
+            include('keyword'),
+
+            (r'%' + identifier, Name.Variable),
+            (r'@' + identifier, Name.Variable.Global),
+            (r'%\d+', Name.Variable.Anonymous),
+            (r'@\d+', Name.Variable.Global),
+            (r'#\d+', Name.Variable.Global),
+            (r'!' + identifier, Name.Variable),
+            (r'!\d+', Name.Variable.Anonymous),
+            (r'c?' + string, String),
+
+            (r'0[xX][a-fA-F0-9]+', Number),
+            (r'-?\d+(?:[.]\d+)?(?:[eE][-+]?\d+(?:[.]\d+)?)?', Number),
+
+            (r'[=<>{}\[\]()*.,!]|x\b', Punctuation)
+        ],
+        'whitespace': [
+            (r'(\n|\s)+', Text),
+            (r';.*?\n', Comment)
+        ],
+        'keyword': [
+            # Regular keywords
+            (words((
+                'begin', 'end', 'true', 'false', 'declare', 'define', 'global',
+                'constant', 'private', 'linker_private', 'internal',
+                'available_externally', 'linkonce', 'linkonce_odr', 'weak',
+                'weak_odr', 'appending', 'dllimport', 'dllexport', 'common',
+                'default', 'hidden', 'protected', 'extern_weak', 'external',
+                'thread_local', 'zeroinitializer', 'undef', 'null', 'to', 'tail',
+                'target', 'triple', 'datalayout', 'volatile', 'nuw', 'nsw', 'nnan',
+                'ninf', 'nsz', 'arcp', 'fast', 'exact', 'inbounds', 'align',
+                'addrspace', 'section', 'alias', 'module', 'asm', 'sideeffect',
+                'gc', 'dbg', 'linker_private_weak', 'attributes', 'blockaddress',
+                'initialexec', 'localdynamic', 'localexec', 'prefix', 'unnamed_addr',
+                'ccc', 'fastcc', 'coldcc', 'x86_stdcallcc', 'x86_fastcallcc',
+                'arm_apcscc', 'arm_aapcscc', 'arm_aapcs_vfpcc', 'ptx_device',
+                'ptx_kernel', 'intel_ocl_bicc', 'msp430_intrcc', 'spir_func',
+                'spir_kernel', 'x86_64_sysvcc', 'x86_64_win64cc', 'x86_thiscallcc',
+                'cc', 'c', 'signext', 'zeroext', 'inreg', 'sret', 'nounwind',
+                'noreturn', 'noalias', 'nocapture', 'byval', 'nest', 'readnone',
+                'readonly', 'inlinehint', 'noinline', 'alwaysinline', 'optsize', 'ssp',
+                'sspreq', 'noredzone', 'noimplicitfloat', 'naked', 'builtin', 'cold',
+                'nobuiltin', 'noduplicate', 'nonlazybind', 'optnone', 'returns_twice',
+                'sanitize_address', 'sanitize_memory', 'sanitize_thread', 'sspstrong',
+                'uwtable', 'returned', 'type', 'opaque', 'eq', 'ne', 'slt', 'sgt',
+                'sle', 'sge', 'ult', 'ugt', 'ule', 'uge', 'oeq', 'one', 'olt', 'ogt',
+                'ole', 'oge', 'ord', 'uno', 'ueq', 'une', 'x', 'acq_rel', 'acquire',
+                'alignstack', 'atomic', 'catch', 'cleanup', 'filter', 'inteldialect',
+                'max', 'min', 'monotonic', 'nand', 'personality', 'release', 'seq_cst',
+                'singlethread', 'umax', 'umin', 'unordered', 'xchg', 'add', 'fadd',
+                'sub', 'fsub', 'mul', 'fmul', 'udiv', 'sdiv', 'fdiv', 'urem', 'srem',
+                'frem', 'shl', 'lshr', 'ashr', 'and', 'or', 'xor', 'icmp', 'fcmp',
+                'phi', 'call', 'trunc', 'zext', 'sext', 'fptrunc', 'fpext', 'uitofp',
+                'sitofp', 'fptoui', 'fptosi', 'inttoptr', 'ptrtoint', 'bitcast',
+                'addrspacecast', 'select', 'va_arg', 'ret', 'br', 'switch', 'invoke',
+                'unwind', 'unreachable', 'indirectbr', 'landingpad', 'resume',
+                'malloc', 'alloca', 'free', 'load', 'store', 'getelementptr',
+                'extractelement', 'insertelement', 'shufflevector', 'getresult',
+                'extractvalue', 'insertvalue', 'atomicrmw', 'cmpxchg', 'fence',
+                'allocsize', 'amdgpu_cs', 'amdgpu_gs', 'amdgpu_kernel', 'amdgpu_ps',
+                'amdgpu_vs', 'any', 'anyregcc', 'argmemonly', 'avr_intrcc',
+                'avr_signalcc', 'caller', 'catchpad', 'catchret', 'catchswitch',
+                'cleanuppad', 'cleanupret', 'comdat', 'convergent', 'cxx_fast_tlscc',
+                'deplibs', 'dereferenceable', 'dereferenceable_or_null', 'distinct',
+                'exactmatch', 'externally_initialized', 'from', 'ghccc', 'hhvm_ccc',
+                'hhvmcc', 'ifunc', 'inaccessiblemem_or_argmemonly', 'inaccessiblememonly',
+                'inalloca', 'jumptable', 'largest', 'local_unnamed_addr', 'minsize',
+                'musttail', 'noduplicates', 'none', 'nonnull', 'norecurse', 'notail',
+                'preserve_allcc', 'preserve_mostcc', 'prologue', 'safestack', 'samesize',
+                'source_filename', 'swiftcc', 'swifterror', 'swiftself', 'webkit_jscc',
+                'within', 'writeonly', 'x86_intrcc', 'x86_vectorcallcc'),
+                suffix=r'\b'), Keyword),
+
+            # Types
+            (words(('void', 'half', 'float', 'double', 'x86_fp80', 'fp128',
+                    'ppc_fp128', 'label', 'metadata', 'token')), Keyword.Type),
+
+            # Integer types
+            (r'i[1-9]\d*', Keyword)
+        ]
+    }
+
+
+class NasmLexer(RegexLexer):
+    """
+    For Nasm (Intel) assembly code.
+    """
+    name = 'NASM'
+    aliases = ['nasm']
+    filenames = ['*.asm', '*.ASM']
+    mimetypes = ['text/x-nasm']
+
+    identifier = r'[a-z$._?][\w$.?#@~]*'
+    hexn = r'(?:0x[0-9a-f]+|$0[0-9a-f]*|[0-9]+[0-9a-f]*h)'
+    octn = r'[0-7]+q'
+    binn = r'[01]+b'
+    decn = r'[0-9]+'
+    floatn = decn + r'\.e?' + decn
+    string = r'"(\\"|[^"\n])*"|' + r"'(\\'|[^'\n])*'|" + r"`(\\`|[^`\n])*`"
+    declkw = r'(?:res|d)[bwdqt]|times'
+    register = (r'r[0-9][0-5]?[bwd]|'
+                r'[a-d][lh]|[er]?[a-d]x|[er]?[sb]p|[er]?[sd]i|[c-gs]s|st[0-7]|'
+                r'mm[0-7]|cr[0-4]|dr[0-367]|tr[3-7]')
+    wordop = r'seg|wrt|strict'
+    type = r'byte|[dq]?word'
+    directives = (r'BITS|USE16|USE32|SECTION|SEGMENT|ABSOLUTE|EXTERN|GLOBAL|'
+                  r'ORG|ALIGN|STRUC|ENDSTRUC|COMMON|CPU|GROUP|UPPERCASE|IMPORT|'
+                  r'EXPORT|LIBRARY|MODULE')
+
+    flags = re.IGNORECASE | re.MULTILINE
+    tokens = {
+        'root': [
+            (r'^\s*%', Comment.Preproc, 'preproc'),
+            include('whitespace'),
+            (identifier + ':', Name.Label),
+            (r'(%s)(\s+)(equ)' % identifier,
+                bygroups(Name.Constant, Keyword.Declaration, Keyword.Declaration),
+                'instruction-args'),
+            (directives, Keyword, 'instruction-args'),
+            (declkw, Keyword.Declaration, 'instruction-args'),
+            (identifier, Name.Function, 'instruction-args'),
+            (r'[\r\n]+', Text)
+        ],
+        'instruction-args': [
+            (string, String),
+            (hexn, Number.Hex),
+            (octn, Number.Oct),
+            (binn, Number.Bin),
+            (floatn, Number.Float),
+            (decn, Number.Integer),
+            include('punctuation'),
+            (register, Name.Builtin),
+            (identifier, Name.Variable),
+            (r'[\r\n]+', Text, '#pop'),
+            include('whitespace')
+        ],
+        'preproc': [
+            (r'[^;\n]+', Comment.Preproc),
+            (r';.*?\n', Comment.Single, '#pop'),
+            (r'\n', Comment.Preproc, '#pop'),
+        ],
+        'whitespace': [
+            (r'\n', Text),
+            (r'[ \t]+', Text),
+            (r';.*', Comment.Single)
+        ],
+        'punctuation': [
+            (r'[,():\[\]]+', Punctuation),
+            (r'[&|^<>+*/%~-]+', Operator),
+            (r'[$]+', Keyword.Constant),
+            (wordop, Operator.Word),
+            (type, Keyword.Type)
+        ],
+    }
+
+
+class NasmObjdumpLexer(ObjdumpLexer):
+    """
+    For the output of 'objdump -d -M intel'.
+
+    .. versionadded:: 2.0
+    """
+    name = 'objdump-nasm'
+    aliases = ['objdump-nasm']
+    filenames = ['*.objdump-intel']
+    mimetypes = ['text/x-nasm-objdump']
+
+    tokens = _objdump_lexer_tokens(NasmLexer)
+
+
+class TasmLexer(RegexLexer):
+    """
+    For Tasm (Turbo Assembler) assembly code.
+    """
+    name = 'TASM'
+    aliases = ['tasm']
+    filenames = ['*.asm', '*.ASM', '*.tasm']
+    mimetypes = ['text/x-tasm']
+
+    identifier = r'[@a-z$._?][\w$.?#@~]*'
+    hexn = r'(?:0x[0-9a-f]+|$0[0-9a-f]*|[0-9]+[0-9a-f]*h)'
+    octn = r'[0-7]+q'
+    binn = r'[01]+b'
+    decn = r'[0-9]+'
+    floatn = decn + r'\.e?' + decn
+    string = r'"(\\"|[^"\n])*"|' + r"'(\\'|[^'\n])*'|" + r"`(\\`|[^`\n])*`"
+    declkw = r'(?:res|d)[bwdqt]|times'
+    register = (r'r[0-9][0-5]?[bwd]|'
+                r'[a-d][lh]|[er]?[a-d]x|[er]?[sb]p|[er]?[sd]i|[c-gs]s|st[0-7]|'
+                r'mm[0-7]|cr[0-4]|dr[0-367]|tr[3-7]')
+    wordop = r'seg|wrt|strict'
+    type = r'byte|[dq]?word'
+    directives = (r'BITS|USE16|USE32|SECTION|SEGMENT|ABSOLUTE|EXTERN|GLOBAL|'
+                  r'ORG|ALIGN|STRUC|ENDSTRUC|ENDS|COMMON|CPU|GROUP|UPPERCASE|INCLUDE|'
+                  r'EXPORT|LIBRARY|MODULE|PROC|ENDP|USES|ARG|DATASEG|UDATASEG|END|IDEAL|'
+                  r'P386|MODEL|ASSUME|CODESEG|SIZE')
+    # T[A-Z][a-z] is more of a convention. Lexer should filter out STRUC definitions
+    # and then 'add' them to datatype somehow.
+    datatype = (r'db|dd|dw|T[A-Z][a-z]+')
+
+    flags = re.IGNORECASE | re.MULTILINE
+    tokens = {
+        'root': [
+            (r'^\s*%', Comment.Preproc, 'preproc'),
+            include('whitespace'),
+            (identifier + ':', Name.Label),
+            (directives, Keyword, 'instruction-args'),
+            (r'(%s)(\s+)(%s)' % (identifier, datatype),
+                bygroups(Name.Constant, Keyword.Declaration, Keyword.Declaration),
+                'instruction-args'),
+            (declkw, Keyword.Declaration, 'instruction-args'),
+            (identifier, Name.Function, 'instruction-args'),
+            (r'[\r\n]+', Text)
+        ],
+        'instruction-args': [
+            (string, String),
+            (hexn, Number.Hex),
+            (octn, Number.Oct),
+            (binn, Number.Bin),
+            (floatn, Number.Float),
+            (decn, Number.Integer),
+            include('punctuation'),
+            (register, Name.Builtin),
+            (identifier, Name.Variable),
+            # Do not match newline when it's preceeded by a backslash
+            (r'(\\\s*)(;.*)([\r\n])', bygroups(Text, Comment.Single, Text)),
+            (r'[\r\n]+', Text, '#pop'),
+            include('whitespace')
+        ],
+        'preproc': [
+            (r'[^;\n]+', Comment.Preproc),
+            (r';.*?\n', Comment.Single, '#pop'),
+            (r'\n', Comment.Preproc, '#pop'),
+        ],
+        'whitespace': [
+            (r'[\n\r]', Text),
+            (r'\\[\n\r]', Text),
+            (r'[ \t]+', Text),
+            (r';.*', Comment.Single)
+        ],
+        'punctuation': [
+            (r'[,():\[\]]+', Punctuation),
+            (r'[&|^<>+*=/%~-]+', Operator),
+            (r'[$]+', Keyword.Constant),
+            (wordop, Operator.Word),
+            (type, Keyword.Type)
+        ],
+    }
+
+
+class Ca65Lexer(RegexLexer):
+    """
+    For ca65 assembler sources.
+
+    .. versionadded:: 1.6
+    """
+    name = 'ca65 assembler'
+    aliases = ['ca65']
+    filenames = ['*.s']
+
+    flags = re.IGNORECASE
+
+    tokens = {
+        'root': [
+            (r';.*', Comment.Single),
+            (r'\s+', Text),
+            (r'[a-z_.@$][\w.@$]*:', Name.Label),
+            (r'((ld|st)[axy]|(in|de)[cxy]|asl|lsr|ro[lr]|adc|sbc|cmp|cp[xy]'
+             r'|cl[cvdi]|se[cdi]|jmp|jsr|bne|beq|bpl|bmi|bvc|bvs|bcc|bcs'
+             r'|p[lh][ap]|rt[is]|brk|nop|ta[xy]|t[xy]a|txs|tsx|and|ora|eor'
+             r'|bit)\b', Keyword),
+            (r'\.\w+', Keyword.Pseudo),
+            (r'[-+~*/^&|!<>=]', Operator),
+            (r'"[^"\n]*.', String),
+            (r"'[^'\n]*.", String.Char),
+            (r'\$[0-9a-f]+|[0-9a-f]+h\b', Number.Hex),
+            (r'\d+', Number.Integer),
+            (r'%[01]+', Number.Bin),
+            (r'[#,.:()=\[\]]', Punctuation),
+            (r'[a-z_.@$][\w.@$]*', Name),
+        ]
+    }
+
+    def analyse_text(self, text):
+        # comments in GAS start with "#"
+        if re.match(r'^\s*;', text, re.MULTILINE):
+            return 0.9

packages/wakatime/packages/pygments/lexers/automation.py

@@ -0,0 +1,374 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.automation
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for automation scripting languages.
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, bygroups, combined
+from pygments.token import Text, Comment, Operator, Name, String, \
+    Number, Punctuation, Generic
+
+__all__ = ['AutohotkeyLexer', 'AutoItLexer']
+
+
+class AutohotkeyLexer(RegexLexer):
+    """
+    For `autohotkey <http://www.autohotkey.com/>`_ source code.
+
+    .. versionadded:: 1.4
+    """
+    name = 'autohotkey'
+    aliases = ['ahk', 'autohotkey']
+    filenames = ['*.ahk', '*.ahkl']
+    mimetypes = ['text/x-autohotkey']
+
+    tokens = {
+        'root': [
+            (r'^(\s*)(/\*)', bygroups(Text, Comment.Multiline), 'incomment'),
+            (r'^(\s*)(\()', bygroups(Text, Generic), 'incontinuation'),
+            (r'\s+;.*?$', Comment.Single),
+            (r'^;.*?$', Comment.Single),
+            (r'[]{}(),;[]', Punctuation),
+            (r'(in|is|and|or|not)\b', Operator.Word),
+            (r'\%[a-zA-Z_#@$][\w#@$]*\%', Name.Variable),
+            (r'!=|==|:=|\.=|<<|>>|[-~+/*%=<>&^|?:!.]', Operator),
+            include('commands'),
+            include('labels'),
+            include('builtInFunctions'),
+            include('builtInVariables'),
+            (r'"', String, combined('stringescape', 'dqs')),
+            include('numbers'),
+            (r'[a-zA-Z_#@$][\w#@$]*', Name),
+            (r'\\|\'', Text),
+            (r'\`([,%`abfnrtv\-+;])', String.Escape),
+            include('garbage'),
+        ],
+        'incomment': [
+            (r'^\s*\*/', Comment.Multiline, '#pop'),
+            (r'[^*/]', Comment.Multiline),
+            (r'[*/]', Comment.Multiline)
+        ],
+        'incontinuation': [
+            (r'^\s*\)', Generic, '#pop'),
+            (r'[^)]', Generic),
+            (r'[)]', Generic),
+        ],
+        'commands': [
+            (r'(?i)^(\s*)(global|local|static|'
+             r'#AllowSameLineComments|#ClipboardTimeout|#CommentFlag|'
+             r'#ErrorStdOut|#EscapeChar|#HotkeyInterval|#HotkeyModifierTimeout|'
+             r'#Hotstring|#IfWinActive|#IfWinExist|#IfWinNotActive|'
+             r'#IfWinNotExist|#IncludeAgain|#Include|#InstallKeybdHook|'
+             r'#InstallMouseHook|#KeyHistory|#LTrim|#MaxHotkeysPerInterval|'
+             r'#MaxMem|#MaxThreads|#MaxThreadsBuffer|#MaxThreadsPerHotkey|'
+             r'#NoEnv|#NoTrayIcon|#Persistent|#SingleInstance|#UseHook|'
+             r'#WinActivateForce|AutoTrim|BlockInput|Break|Click|ClipWait|'
+             r'Continue|Control|ControlClick|ControlFocus|ControlGetFocus|'
+             r'ControlGetPos|ControlGetText|ControlGet|ControlMove|ControlSend|'
+             r'ControlSendRaw|ControlSetText|CoordMode|Critical|'
+             r'DetectHiddenText|DetectHiddenWindows|Drive|DriveGet|'
+             r'DriveSpaceFree|Edit|Else|EnvAdd|EnvDiv|EnvGet|EnvMult|EnvSet|'
+             r'EnvSub|EnvUpdate|Exit|ExitApp|FileAppend|'
+             r'FileCopy|FileCopyDir|FileCreateDir|FileCreateShortcut|'
+             r'FileDelete|FileGetAttrib|FileGetShortcut|FileGetSize|'
+             r'FileGetTime|FileGetVersion|FileInstall|FileMove|FileMoveDir|'
+             r'FileRead|FileReadLine|FileRecycle|FileRecycleEmpty|'
+             r'FileRemoveDir|FileSelectFile|FileSelectFolder|FileSetAttrib|'
+             r'FileSetTime|FormatTime|GetKeyState|Gosub|Goto|GroupActivate|'
+             r'GroupAdd|GroupClose|GroupDeactivate|Gui|GuiControl|'
+             r'GuiControlGet|Hotkey|IfEqual|IfExist|IfGreaterOrEqual|IfGreater|'
+             r'IfInString|IfLess|IfLessOrEqual|IfMsgBox|IfNotEqual|IfNotExist|'
+             r'IfNotInString|IfWinActive|IfWinExist|IfWinNotActive|'
+             r'IfWinNotExist|If |ImageSearch|IniDelete|IniRead|IniWrite|'
+             r'InputBox|Input|KeyHistory|KeyWait|ListHotkeys|ListLines|'
+             r'ListVars|Loop|Menu|MouseClickDrag|MouseClick|MouseGetPos|'
+             r'MouseMove|MsgBox|OnExit|OutputDebug|Pause|PixelGetColor|'
+             r'PixelSearch|PostMessage|Process|Progress|Random|RegDelete|'
+             r'RegRead|RegWrite|Reload|Repeat|Return|RunAs|RunWait|Run|'
+             r'SendEvent|SendInput|SendMessage|SendMode|SendPlay|SendRaw|Send|'
+             r'SetBatchLines|SetCapslockState|SetControlDelay|'
+             r'SetDefaultMouseSpeed|SetEnv|SetFormat|SetKeyDelay|'
+             r'SetMouseDelay|SetNumlockState|SetScrollLockState|'
+             r'SetStoreCapslockMode|SetTimer|SetTitleMatchMode|'
+             r'SetWinDelay|SetWorkingDir|Shutdown|Sleep|Sort|SoundBeep|'
+             r'SoundGet|SoundGetWaveVolume|SoundPlay|SoundSet|'
+             r'SoundSetWaveVolume|SplashImage|SplashTextOff|SplashTextOn|'
+             r'SplitPath|StatusBarGetText|StatusBarWait|StringCaseSense|'
+             r'StringGetPos|StringLeft|StringLen|StringLower|StringMid|'
+             r'StringReplace|StringRight|StringSplit|StringTrimLeft|'
+             r'StringTrimRight|StringUpper|Suspend|SysGet|Thread|ToolTip|'
+             r'Transform|TrayTip|URLDownloadToFile|While|WinActivate|'
+             r'WinActivateBottom|WinClose|WinGetActiveStats|WinGetActiveTitle|'
+             r'WinGetClass|WinGetPos|WinGetText|WinGetTitle|WinGet|WinHide|'
+             r'WinKill|WinMaximize|WinMenuSelectItem|WinMinimizeAllUndo|'
+             r'WinMinimizeAll|WinMinimize|WinMove|WinRestore|WinSetTitle|'
+             r'WinSet|WinShow|WinWaitActive|WinWaitClose|WinWaitNotActive|'
+             r'WinWait)\b', bygroups(Text, Name.Builtin)),
+        ],
+        'builtInFunctions': [
+            (r'(?i)(Abs|ACos|Asc|ASin|ATan|Ceil|Chr|Cos|DllCall|Exp|FileExist|'
+             r'Floor|GetKeyState|IL_Add|IL_Create|IL_Destroy|InStr|IsFunc|'
+             r'IsLabel|Ln|Log|LV_Add|LV_Delete|LV_DeleteCol|LV_GetCount|'
+             r'LV_GetNext|LV_GetText|LV_Insert|LV_InsertCol|LV_Modify|'
+             r'LV_ModifyCol|LV_SetImageList|Mod|NumGet|NumPut|OnMessage|'
+             r'RegExMatch|RegExReplace|RegisterCallback|Round|SB_SetIcon|'
+             r'SB_SetParts|SB_SetText|Sin|Sqrt|StrLen|SubStr|Tan|TV_Add|'
+             r'TV_Delete|TV_GetChild|TV_GetCount|TV_GetNext|TV_Get|'
+             r'TV_GetParent|TV_GetPrev|TV_GetSelection|TV_GetText|TV_Modify|'
+             r'VarSetCapacity|WinActive|WinExist|Object|ComObjActive|'
+             r'ComObjArray|ComObjEnwrap|ComObjUnwrap|ComObjParameter|'
+             r'ComObjType|ComObjConnect|ComObjCreate|ComObjGet|ComObjError|'
+             r'ComObjValue|Insert|MinIndex|MaxIndex|Remove|SetCapacity|'
+             r'GetCapacity|GetAddress|_NewEnum|FileOpen|Read|Write|ReadLine|'
+             r'WriteLine|ReadNumType|WriteNumType|RawRead|RawWrite|Seek|Tell|'
+             r'Close|Next|IsObject|StrPut|StrGet|Trim|LTrim|RTrim)\b',
+             Name.Function),
+        ],
+        'builtInVariables': [
+            (r'(?i)(A_AhkPath|A_AhkVersion|A_AppData|A_AppDataCommon|'
+             r'A_AutoTrim|A_BatchLines|A_CaretX|A_CaretY|A_ComputerName|'
+             r'A_ControlDelay|A_Cursor|A_DDDD|A_DDD|A_DD|A_DefaultMouseSpeed|'
+             r'A_Desktop|A_DesktopCommon|A_DetectHiddenText|'
+             r'A_DetectHiddenWindows|A_EndChar|A_EventInfo|A_ExitReason|'
+             r'A_FormatFloat|A_FormatInteger|A_Gui|A_GuiEvent|A_GuiControl|'
+             r'A_GuiControlEvent|A_GuiHeight|A_GuiWidth|A_GuiX|A_GuiY|A_Hour|'
+             r'A_IconFile|A_IconHidden|A_IconNumber|A_IconTip|A_Index|'
+             r'A_IPAddress1|A_IPAddress2|A_IPAddress3|A_IPAddress4|A_ISAdmin|'
+             r'A_IsCompiled|A_IsCritical|A_IsPaused|A_IsSuspended|A_KeyDelay|'
+             r'A_Language|A_LastError|A_LineFile|A_LineNumber|A_LoopField|'
+             r'A_LoopFileAttrib|A_LoopFileDir|A_LoopFileExt|A_LoopFileFullPath|'
+             r'A_LoopFileLongPath|A_LoopFileName|A_LoopFileShortName|'
+             r'A_LoopFileShortPath|A_LoopFileSize|A_LoopFileSizeKB|'
+             r'A_LoopFileSizeMB|A_LoopFileTimeAccessed|A_LoopFileTimeCreated|'
+             r'A_LoopFileTimeModified|A_LoopReadLine|A_LoopRegKey|'
+             r'A_LoopRegName|A_LoopRegSubkey|A_LoopRegTimeModified|'
+             r'A_LoopRegType|A_MDAY|A_Min|A_MM|A_MMM|A_MMMM|A_Mon|A_MouseDelay|'
+             r'A_MSec|A_MyDocuments|A_Now|A_NowUTC|A_NumBatchLines|A_OSType|'
+             r'A_OSVersion|A_PriorHotkey|A_ProgramFiles|A_Programs|'
+             r'A_ProgramsCommon|A_ScreenHeight|A_ScreenWidth|A_ScriptDir|'
+             r'A_ScriptFullPath|A_ScriptName|A_Sec|A_Space|A_StartMenu|'
+             r'A_StartMenuCommon|A_Startup|A_StartupCommon|A_StringCaseSense|'
+             r'A_Tab|A_Temp|A_ThisFunc|A_ThisHotkey|A_ThisLabel|A_ThisMenu|'
+             r'A_ThisMenuItem|A_ThisMenuItemPos|A_TickCount|A_TimeIdle|'
+             r'A_TimeIdlePhysical|A_TimeSincePriorHotkey|A_TimeSinceThisHotkey|'
+             r'A_TitleMatchMode|A_TitleMatchModeSpeed|A_UserName|A_WDay|'
+             r'A_WinDelay|A_WinDir|A_WorkingDir|A_YDay|A_YEAR|A_YWeek|A_YYYY|'
+             r'Clipboard|ClipboardAll|ComSpec|ErrorLevel|ProgramFiles|True|'
+             r'False|A_IsUnicode|A_FileEncoding|A_OSVersion|A_PtrSize)\b',
+             Name.Variable),
+        ],
+        'labels': [
+            # hotkeys and labels
+            # technically, hotkey names are limited to named keys and buttons
+            (r'(^\s*)([^:\s("]+?:{1,2})', bygroups(Text, Name.Label)),
+            (r'(^\s*)(::[^:\s]+?::)', bygroups(Text, Name.Label)),
+        ],
+        'numbers': [
+            (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
+            (r'\d+[eE][+-]?[0-9]+', Number.Float),
+            (r'0\d+', Number.Oct),
+            (r'0[xX][a-fA-F0-9]+', Number.Hex),
+            (r'\d+L', Number.Integer.Long),
+            (r'\d+', Number.Integer)
+        ],
+        'stringescape': [
+            (r'\"\"|\`([,%`abfnrtv])', String.Escape),
+        ],
+        'strings': [
+            (r'[^"\n]+', String),
+        ],
+        'dqs': [
+            (r'"', String, '#pop'),
+            include('strings')
+        ],
+        'garbage': [
+            (r'[^\S\n]', Text),
+            # (r'.', Text),      # no cheating
+        ],
+    }
+
+
+class AutoItLexer(RegexLexer):
+    """
+    For `AutoIt <http://www.autoitscript.com/site/autoit/>`_ files.
+
+    AutoIt is a freeware BASIC-like scripting language
+    designed for automating the Windows GUI and general scripting
+
+    .. versionadded:: 1.6
+    """
+    name = 'AutoIt'
+    aliases = ['autoit']
+    filenames = ['*.au3']
+    mimetypes = ['text/x-autoit']
+
+    # Keywords, functions, macros from au3.keywords.properties
+    # which can be found in AutoIt installed directory, e.g.
+    # c:\Program Files (x86)\AutoIt3\SciTE\au3.keywords.properties
+
+    keywords = """\
+    #include-once #include #endregion #forcedef #forceref #region
+    and byref case continueloop dim do else elseif endfunc endif
+    endselect exit exitloop for func global
+    if local next not or return select step
+    then to until wend while exit""".split()
+
+    functions = """\
+    abs acos adlibregister adlibunregister asc ascw asin assign atan
+    autoitsetoption autoitwingettitle autoitwinsettitle beep binary binarylen
+    binarymid binarytostring bitand bitnot bitor bitrotate bitshift bitxor
+    blockinput break call cdtray ceiling chr chrw clipget clipput consoleread
+    consolewrite consolewriteerror controlclick controlcommand controldisable
+    controlenable controlfocus controlgetfocus controlgethandle controlgetpos
+    controlgettext controlhide controllistview controlmove controlsend
+    controlsettext controlshow controltreeview cos dec dircopy dircreate
+    dirgetsize dirmove dirremove dllcall dllcalladdress dllcallbackfree
+    dllcallbackgetptr dllcallbackregister dllclose dllopen dllstructcreate
+    dllstructgetdata dllstructgetptr dllstructgetsize dllstructsetdata
+    drivegetdrive drivegetfilesystem drivegetlabel drivegetserial drivegettype
+    drivemapadd drivemapdel drivemapget drivesetlabel drivespacefree
+    drivespacetotal drivestatus envget envset envupdate eval execute exp
+    filechangedir fileclose filecopy filecreatentfslink filecreateshortcut
+    filedelete fileexists filefindfirstfile filefindnextfile fileflush
+    filegetattrib filegetencoding filegetlongname filegetpos filegetshortcut
+    filegetshortname filegetsize filegettime filegetversion fileinstall filemove
+    fileopen fileopendialog fileread filereadline filerecycle filerecycleempty
+    filesavedialog fileselectfolder filesetattrib filesetpos filesettime
+    filewrite filewriteline floor ftpsetproxy guicreate guictrlcreateavi
+    guictrlcreatebutton guictrlcreatecheckbox guictrlcreatecombo
+    guictrlcreatecontextmenu guictrlcreatedate guictrlcreatedummy
+    guictrlcreateedit guictrlcreategraphic guictrlcreategroup guictrlcreateicon
+    guictrlcreateinput guictrlcreatelabel guictrlcreatelist
+    guictrlcreatelistview guictrlcreatelistviewitem guictrlcreatemenu
+    guictrlcreatemenuitem guictrlcreatemonthcal guictrlcreateobj
+    guictrlcreatepic guictrlcreateprogress guictrlcreateradio
+    guictrlcreateslider guictrlcreatetab guictrlcreatetabitem
+    guictrlcreatetreeview guictrlcreatetreeviewitem guictrlcreateupdown
+    guictrldelete guictrlgethandle guictrlgetstate guictrlread guictrlrecvmsg
+    guictrlregisterlistviewsort guictrlsendmsg guictrlsendtodummy
+    guictrlsetbkcolor guictrlsetcolor guictrlsetcursor guictrlsetdata
+    guictrlsetdefbkcolor guictrlsetdefcolor guictrlsetfont guictrlsetgraphic
+    guictrlsetimage guictrlsetlimit guictrlsetonevent guictrlsetpos
+    guictrlsetresizing guictrlsetstate guictrlsetstyle guictrlsettip guidelete
+    guigetcursorinfo guigetmsg guigetstyle guiregistermsg guisetaccelerators
+    guisetbkcolor guisetcoord guisetcursor guisetfont guisethelp guiseticon
+    guisetonevent guisetstate guisetstyle guistartgroup guiswitch hex hotkeyset
+    httpsetproxy httpsetuseragent hwnd inetclose inetget inetgetinfo inetgetsize
+    inetread inidelete iniread inireadsection inireadsectionnames
+    inirenamesection iniwrite iniwritesection inputbox int isadmin isarray
+    isbinary isbool isdeclared isdllstruct isfloat ishwnd isint iskeyword
+    isnumber isobj isptr isstring log memgetstats mod mouseclick mouseclickdrag
+    mousedown mousegetcursor mousegetpos mousemove mouseup mousewheel msgbox
+    number objcreate objcreateinterface objevent objevent objget objname
+    onautoitexitregister onautoitexitunregister opt ping pixelchecksum
+    pixelgetcolor pixelsearch pluginclose pluginopen processclose processexists
+    processgetstats processlist processsetpriority processwait processwaitclose
+    progressoff progresson progressset ptr random regdelete regenumkey
+    regenumval regread regwrite round run runas runaswait runwait send
+    sendkeepactive seterror setextended shellexecute shellexecutewait shutdown
+    sin sleep soundplay soundsetwavevolume splashimageon splashoff splashtexton
+    sqrt srandom statusbargettext stderrread stdinwrite stdioclose stdoutread
+    string stringaddcr stringcompare stringformat stringfromasciiarray
+    stringinstr stringisalnum stringisalpha stringisascii stringisdigit
+    stringisfloat stringisint stringislower stringisspace stringisupper
+    stringisxdigit stringleft stringlen stringlower stringmid stringregexp
+    stringregexpreplace stringreplace stringright stringsplit stringstripcr
+    stringstripws stringtoasciiarray stringtobinary stringtrimleft
+    stringtrimright stringupper tan tcpaccept tcpclosesocket tcpconnect
+    tcplisten tcpnametoip tcprecv tcpsend tcpshutdown tcpstartup timerdiff
+    timerinit tooltip traycreateitem traycreatemenu traygetmsg trayitemdelete
+    trayitemgethandle trayitemgetstate trayitemgettext trayitemsetonevent
+    trayitemsetstate trayitemsettext traysetclick trayseticon traysetonevent
+    traysetpauseicon traysetstate traysettooltip traytip ubound udpbind
+    udpclosesocket udpopen udprecv udpsend udpshutdown udpstartup vargettype
+    winactivate winactive winclose winexists winflash wingetcaretpos
+    wingetclasslist wingetclientsize wingethandle wingetpos wingetprocess
+    wingetstate wingettext wingettitle winkill winlist winmenuselectitem
+    winminimizeall winminimizeallundo winmove winsetontop winsetstate
+    winsettitle winsettrans winwait winwaitactive winwaitclose
+    winwaitnotactive""".split()
+
+    macros = """\
+    @appdatacommondir @appdatadir @autoitexe @autoitpid @autoitversion
+    @autoitx64 @com_eventobj @commonfilesdir @compiled @computername @comspec
+    @cpuarch @cr @crlf @desktopcommondir @desktopdepth @desktopdir
+    @desktopheight @desktoprefresh @desktopwidth @documentscommondir @error
+    @exitcode @exitmethod @extended @favoritescommondir @favoritesdir
+    @gui_ctrlhandle @gui_ctrlid @gui_dragfile @gui_dragid @gui_dropid
+    @gui_winhandle @homedrive @homepath @homeshare @hotkeypressed @hour
+    @ipaddress1 @ipaddress2 @ipaddress3 @ipaddress4 @kblayout @lf
+    @logondnsdomain @logondomain @logonserver @mday @min @mon @msec @muilang
+    @mydocumentsdir @numparams @osarch @osbuild @oslang @osservicepack @ostype
+    @osversion @programfilesdir @programscommondir @programsdir @scriptdir
+    @scriptfullpath @scriptlinenumber @scriptname @sec @startmenucommondir
+    @startmenudir @startupcommondir @startupdir @sw_disable @sw_enable @sw_hide
+    @sw_lock @sw_maximize @sw_minimize @sw_restore @sw_show @sw_showdefault
+    @sw_showmaximized @sw_showminimized @sw_showminnoactive @sw_showna
+    @sw_shownoactivate @sw_shownormal @sw_unlock @systemdir @tab @tempdir
+    @tray_id @trayiconflashing @trayiconvisible @username @userprofiledir @wday
+    @windowsdir @workingdir @yday @year""".split()
+
+    tokens = {
+        'root': [
+            (r';.*\n', Comment.Single),
+            (r'(#comments-start|#cs)(.|\n)*?(#comments-end|#ce)',
+             Comment.Multiline),
+            (r'[\[\]{}(),;]', Punctuation),
+            (r'(and|or|not)\b', Operator.Word),
+            (r'[$|@][a-zA-Z_]\w*', Name.Variable),
+            (r'!=|==|:=|\.=|<<|>>|[-~+/*%=<>&^|?:!.]', Operator),
+            include('commands'),
+            include('labels'),
+            include('builtInFunctions'),
+            include('builtInMarcros'),
+            (r'"', String, combined('stringescape', 'dqs')),
+            include('numbers'),
+            (r'[a-zA-Z_#@$][\w#@$]*', Name),
+            (r'\\|\'', Text),
+            (r'\`([,%`abfnrtv\-+;])', String.Escape),
+            (r'_\n', Text),  # Line continuation
+            include('garbage'),
+        ],
+        'commands': [
+            (r'(?i)(\s*)(%s)\b' % '|'.join(keywords),
+             bygroups(Text, Name.Builtin)),
+        ],
+        'builtInFunctions': [
+            (r'(?i)(%s)\b' % '|'.join(functions),
+             Name.Function),
+        ],
+        'builtInMarcros': [
+            (r'(?i)(%s)\b' % '|'.join(macros),
+             Name.Variable.Global),
+        ],
+        'labels': [
+            # sendkeys
+            (r'(^\s*)(\{\S+?\})', bygroups(Text, Name.Label)),
+        ],
+        'numbers': [
+            (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
+            (r'\d+[eE][+-]?[0-9]+', Number.Float),
+            (r'0\d+', Number.Oct),
+            (r'0[xX][a-fA-F0-9]+', Number.Hex),
+            (r'\d+L', Number.Integer.Long),
+            (r'\d+', Number.Integer)
+        ],
+        'stringescape': [
+            (r'\"\"|\`([,%`abfnrtv])', String.Escape),
+        ],
+        'strings': [
+            (r'[^"\n]+', String),
+        ],
+        'dqs': [
+            (r'"', String, '#pop'),
+            include('strings')
+        ],
+        'garbage': [
+            (r'[^\S\n]', Text),
+        ],
+    }