Compare commits

...

193 Commits

Author SHA1 Message Date
a387c08b44 changes for v11.1.0 2022-11-11 21:57:09 -03:00
3c2947cf79 Support for api key vault cmd config 2022-11-11 21:55:29 -03:00
54e6772a80 v11.0.8 2022-08-23 15:14:53 +02:00
b576dfafe6 changes for v11.0.8 2022-08-23 15:14:40 +02:00
5299efd6fa sometimes selection object can have no elements but eval truthy 2022-08-23 15:13:44 +02:00
74583a6845 v11.0.7 2022-06-25 13:46:38 -03:00
31f1f8ecdc changes for v11.0.7 2022-06-25 13:46:25 -03:00
c1f58fd05d check wakatime-cli versions in background thread to prevent blocking UI 2022-06-25 13:45:29 -03:00
28063e3ac4 v11.0.6 2022-06-08 07:44:41 -04:00
4d56aca1a1 changes for v11.0.6 2022-06-08 07:44:01 -04:00
e15c514ef3 Fix call to log helper 2022-06-08 07:43:01 -04:00
fe582c84b9 v11.0.5 2022-04-29 11:19:26 -07:00
d7ee8675d8 changes for v11.0.5 2022-04-29 11:19:05 -07:00
da17125b97 Prevent installing global url opener 2022-04-29 11:14:55 -07:00
847223cdce Clean up legacy wakatime-cli directory if exists 2022-02-21 20:08:48 -08:00
b405f99cff chmod wakatime-cli after copying 2022-01-21 15:12:51 -08:00
b5210b77ce v11.0.4 2022-01-06 18:40:32 -08:00
fac1192228 changes for v11.0.4 2022-01-06 18:40:18 -08:00
4d6533b2ee fix lineno, cursorpos, and lines-in-file arguments 2022-01-06 18:37:52 -08:00
7d2bd0b7c5 Copy wakatime-cli when symlink fails 2022-01-06 18:16:02 -08:00
66f5f48f33 create symlink to wakatime-cli 2022-01-06 10:23:09 -08:00
e337afcc53 Fix Popen override 2022-01-06 07:56:45 -08:00
033c07f070 prevent using empty cached version 2022-01-05 04:15:10 -08:00
c09767b58d v11.0.3 2021-12-31 16:55:16 -08:00
ac80c4268b changes for v11.0.3 2021-12-31 16:55:05 -08:00
e5331d3086 only remove wakatime-cli after finished downloading new one 2021-12-31 16:42:11 -08:00
bebb46dda6 v11.0.2 2021-11-17 14:01:24 -08:00
9767790063 changes for v11.0.2 2021-11-17 14:01:08 -08:00
75c219055d encode cursorpos in json as int not str 2021-11-17 13:52:49 -08:00
ef77e1f178 v11.0.1 2021-11-16 09:47:29 -08:00
4025decc12 changes for v11.0.1 2021-11-16 09:47:14 -08:00
086c700151 turn off strict config parsing in py3 2021-11-16 09:43:20 -08:00
650bb6fa26 log error response when fail to check wakatime-cli version or download binary 2021-11-01 17:27:12 -07:00
389c84673e retry httperrors in py2 without ssl 2021-11-01 17:18:06 -07:00
6fa1321a95 v11.0.0 2021-10-31 12:59:57 -07:00
f1a8fcab44 changes for v11.0.0 2021-10-31 12:59:36 -07:00
3937b083c5 Finish implementation of using new Go wakatime-cli 2021-10-31 12:57:28 -07:00
711aab0d18 Remove unused python packages folder 2021-10-31 11:42:11 -07:00
cb8ce3a54e Switch to new Go wakatime-cli 2021-10-31 11:41:21 -07:00
7db5fe0a5d strip newlines from today output before displaying in status bar 2021-10-30 15:18:26 -07:00
cebcfaa0e9 Merge pull request #106 from jamesa/master
Fix Python 3.8 warning when getting Unix time zone
2021-10-25 21:32:02 -07:00
f8faed6e47 Fix Python 3.8 warning when getting Unix time zone 2021-10-25 17:19:39 -04:00
fb303e048f v10.0.1 2020-12-28 18:54:50 -08:00
4f11222c2b changes for v10.0.1 2020-12-28 18:54:37 -08:00
72b72dc9f0 add sublime text to subtitle 2020-12-28 18:54:07 -08:00
1b07d0442b v10.0.0 2020-12-28 18:46:13 -08:00
fbd8e84ea1 changes for v10.0.0 2020-12-28 18:44:55 -08:00
01c0e7758e Partially revert "Use wakatime-cli standalone" to support both
standalone and Python source wakatime-cli versions.

This partially reverts commit d588451468.
2020-12-28 18:42:00 -08:00
28556de3b6 add back python binary location setting 2020-12-28 17:57:46 -08:00
809e43cfe5 fix link to api key 2020-12-28 17:49:31 -08:00
22ddbe27b0 link directly to wakatime install instructions 2020-12-28 17:44:00 -08:00
ddaf60b8b0 remove ide setting files from revision control 2020-07-30 15:53:04 -07:00
2e6a87c67e prevent executing wakatime-cli before it has been downloaded 2020-03-01 10:01:52 -08:00
01503b1c20 Merge pull request #99 from gandarez/standalone
Use wakatime-cli standalone
2020-03-01 10:57:14 -05:00
5a4ac9c11d check if cli is installed before sending heartbeat 2020-02-29 20:01:34 -05:00
fa0a3aacb5 add api_key into sublime-settings 2020-02-29 19:22:32 -05:00
d588451468 Use wakatime-cli standalone 2020-02-29 19:15:40 -05:00
483d8f596e v9.1.2 2020-02-13 09:24:11 -08:00
03f2d6d580 changes for v9.1.2 2020-02-13 09:23:58 -08:00
e1390d7647 Upgrade wakatime-cli to v13.0.7 2020-02-13 09:22:15 -08:00
c87bdd041c Upgrade wakatime-cli to v13.0.5 2020-02-11 21:26:01 -08:00
3a65395636 v9.1.1 2020-02-11 08:12:07 -08:00
0b2f3aa9a4 changes for v9.1.1 2020-02-11 08:11:52 -08:00
58ef2cd794 fix typo 2020-02-11 08:10:39 -08:00
04173d3bcc v9.1.0 2020-02-09 23:37:17 -08:00
3206a07476 changes for v9.1.0 2020-02-09 23:36:29 -08:00
9330236816 upgrade wakatime-cli to v13.0.4 2020-02-09 23:35:43 -08:00
885c11f01a Detect python in Windows LocalAppData install locations 2020-02-09 23:33:39 -08:00
8acda0157a upgrade embedded python to v3.8.1 2020-02-09 23:17:13 -08:00
935ddbd5f6 v9.0.2 2019-12-04 22:04:13 -08:00
b57b1eb696 changes for v9.0.2 2019-12-04 22:03:49 -08:00
6ec097b9d1 upgrade wakatime-cli to v13.0.3 2019-12-04 22:02:51 -08:00
b3ed36d3b2 fix badge locations 2019-11-27 22:28:20 -08:00
3669e4df6a add wakatime badge 2019-11-25 21:30:54 -08:00
3504096082 v9.0.1 2019-11-24 07:49:27 -08:00
5990947706 changes for v9.0.1 2019-11-24 07:48:58 -08:00
2246e31244 upgrade wakatime-cli to v13.0.2 2019-11-24 07:46:13 -08:00
b55fe702d3 v9.0.0 2019-06-23 19:03:26 -07:00
e0fbbb50bb changes for v9.0.0 2019-06-23 19:03:08 -07:00
32c0cb5a97 upgrade wakatime-cli to v12.0.0 2019-06-23 18:56:57 -07:00
67d8b0d24f v8.7.0 2019-05-29 06:44:07 -07:00
b8b2f4944b changes for v8.7.0 2019-05-29 06:43:54 -07:00
a20161164c prevent creating user settings file when api key found from common config 2019-05-29 06:42:38 -07:00
405211bb07 v8.6.1 2019-05-28 20:54:57 -07:00
ffc879c4eb changes for v8.6.1 2019-05-28 20:54:43 -07:00
1e23919694 fix import path of parseConfigFile 2019-05-28 20:53:58 -07:00
b2086a3cd2 v8.6.0 2019-05-27 12:41:14 -07:00
005b07520c changes for v8.6.0 2019-05-27 12:40:55 -07:00
60608bd322 Skip prompting for api key when found from common config file 2019-05-27 12:39:48 -07:00
cde8f8f1de v8.5.0 2019-05-10 09:45:11 -07:00
4adfca154c changes for v8.5.0 2019-05-10 09:44:57 -07:00
f7b3924a30 use wakatime-cli to fetch today coding time 2019-05-10 09:43:30 -07:00
db00024455 remove clock icon from status bar 2019-05-10 09:18:38 -07:00
9a6be7ca4e v8.4.2 2019-05-07 18:44:24 -07:00
1ea9b2a761 changes for v8.4.2 2019-05-07 18:43:57 -07:00
bd5e87e030 upgrade wakatime-cli to v11.0.0 2019-05-07 18:42:31 -07:00
0256ff4a6a v8.4.1 2019-05-01 10:57:50 -07:00
9d170b3276 changes for v8.4.1 2019-05-01 10:57:38 -07:00
c54e575210 use api subdomain for fetching summaries 2019-05-01 10:57:11 -07:00
07513d8f10 v8.4.0 2019-05-01 10:49:59 -07:00
30902cc050 changes for v8.4.0 2019-05-01 10:49:40 -07:00
aa7962d49a show today coding activity time in status bar 2019-05-01 10:48:28 -07:00
d8c662f3db v8.3.6 2019-04-30 16:25:28 -07:00
10d88ebf2d changes for v8.3.6 2019-04-30 16:25:14 -07:00
2f28c561b1 upgrade wakatime-cli to v10.8.4 2019-04-30 16:24:35 -07:00
24968507df v8.3.5 2019-04-30 09:21:47 -07:00
641cd539ed changes for v8.3.5 2019-04-30 09:21:33 -07:00
0c65d7e5b2 Upgrade wakatime-cli to v10.8.3 2019-04-30 09:20:29 -07:00
f0532f5b8e v8.3.4 2019-03-30 19:17:06 -07:00
8094db9680 changes for v8.3.4 2019-03-30 19:16:49 -07:00
bf20551849 upgrade wakatime-cli to v10.8.2 2019-03-30 18:53:40 -07:00
2b6e32b578 remove redundant line 2019-03-13 09:36:51 +09:00
363c3d38e2 update license file with copyright owner inline 2019-03-13 09:20:50 +09:00
88466d7db2 v8.3.3 2018-12-19 07:43:01 -08:00
122fcbbee5 changes for v8.3.3 2018-12-19 07:42:30 -08:00
c41fcec5d8 upgrade wakatime-cli to v10.6.1 2018-12-19 07:38:18 -08:00
be09b34d44 v8.3.2 2018-10-06 20:40:57 -07:00
e1ee1c1216 changes for v8.3.2 2018-10-06 20:40:37 -07:00
a37061924b Send buffered heartbeats to API every 30 seconds. 2018-10-06 20:40:12 -07:00
da01fa268b v8.3.1 2018-10-05 00:07:11 -07:00
c279418651 changes for v8.3.1 2018-10-05 00:06:46 -07:00
5cf2c8f7ac upgrade wakatime-cli to v10.4.1 2018-10-05 00:06:02 -07:00
d1455e77a8 v8.3.0 2018-10-03 00:55:02 -07:00
8499e7bafe changes for v8.3.0 2018-10-03 00:54:40 -07:00
abc26a0864 upgrade wakatime-cli to v10.4.0 2018-10-03 00:47:20 -07:00
71ad97ffe9 v8.2.0 2018-09-30 22:03:01 -07:00
3ec5995c99 changes for v8.2.0 2018-09-30 22:02:53 -07:00
195cf4de36 upgrade wakatime-cli to v10.3.0 2018-09-30 21:48:20 -07:00
b39eefb4f5 cross-platform Popen with hidden window 2018-09-30 21:29:04 -07:00
bbf5761e26 v8.1.2 2018-09-20 22:31:14 -07:00
c4df1dc633 changes for v8.1.2 2018-09-20 22:30:57 -07:00
360a491cda upgrade wakatime-cli to v10.2.4 2018-09-20 22:29:34 -07:00
f61a34eda7 build subprocess stdin near heartbeats 2018-09-13 19:39:05 -07:00
48123d7409 v8.1.1 2018-04-26 08:42:51 -07:00
c8a15d7ac0 changes for v8.1.1 2018-04-26 08:42:00 -07:00
202df81e04 upgrade wakatime-cli to v10.2.1 2018-04-26 08:40:02 -07:00
5e34f3f6a7 v8.1.0 2018-04-03 23:43:43 -07:00
d4441e5575 changes for v8.1.0 2018-04-03 23:43:26 -07:00
9eac8e2bd3 prefer python3 when running wakatime-cli 2018-04-03 23:42:02 -07:00
11d8fc3a09 v8.0.8 2018-03-15 01:51:31 -07:00
d1f1f51f23 changes for v8.0.8 2018-03-15 01:51:20 -07:00
b10bb36c09 Upgrade wakatime-cli to v10.1.3 2018-03-15 01:50:36 -07:00
dc9474befa v8.0.7 2018-03-15 01:32:56 -07:00
b910807e98 changes for v8.0.7 2018-03-15 01:32:37 -07:00
bc770515f0 Upgrade wakatime-cli to v10.1.2 2018-03-15 01:31:17 -07:00
9e102d7c5c v8.0.6 2018-01-04 23:34:40 -08:00
5c1770fb48 changes for v8.0.6 2018-01-04 23:34:13 -08:00
683397534c upgrade wakatime-cli to v10.1.0 2018-01-04 23:33:07 -08:00
1c92017543 v8.0.5 2017-11-24 16:16:47 -08:00
fda1307668 changes for v8.0.5 2017-11-24 16:16:34 -08:00
1c84d457c5 upgrade wakatime-cli to v10.0.5 2017-11-24 16:16:04 -08:00
1e680ce739 v8.0.4 2017-11-23 12:49:07 -08:00
376adbb7d7 changes for v8.0.4 2017-11-23 12:48:44 -08:00
e0040e185b upgrade wakatime-cli to v10.0.4 2017-11-23 12:41:59 -08:00
c4a88541d0 v8.0.3 2017-11-22 13:12:09 -08:00
0cf621d177 changes for v8.0.3 2017-11-22 13:11:48 -08:00
db9d6cec97 upgrade wakatime-cli to v10.0.3 2017-11-22 13:09:17 -08:00
2c17f49a6b v8.0.2 2017-11-15 18:36:43 -08:00
95116d6007 changes for v8.0.2 2017-11-15 18:36:28 -08:00
8c52596f8f upgrade wakatime-cli to v10.0.2 2017-11-15 18:35:43 -08:00
3109817dc7 v8.0.1 2017-11-09 09:10:19 -08:00
0c0f965763 changes for v8.0.1 2017-11-09 09:10:06 -08:00
1573e9c825 upgrade wakatime-cli to v10.0.1 2017-11-09 09:09:28 -08:00
a0b8f349c2 v8.0.0 2017-11-08 23:14:04 -08:00
2fb60b1589 changes for v8.0.0 2017-11-08 23:13:47 -08:00
02786a744e upgrade wakatime-cli to v10.0.0 2017-11-08 23:12:05 -08:00
729a4360ba v7.0.26 2017-11-07 18:55:25 -08:00
8f45de85ec changes for v7.0.26 2017-11-07 18:55:03 -08:00
4672f70c87 upgrade wakatime-cli to v9.0.1 2017-11-07 18:54:17 -08:00
46a9aae942 v7.0.25 2017-11-05 20:10:51 -08:00
9e77ce2697 changes for v7.0.25 2017-11-05 20:08:25 -08:00
385ba818cc ability to override python binary location 2017-11-05 20:04:48 -08:00
7492c3ce12 upgrade wakatime-cli to v9.0.0 2017-11-05 19:51:43 -08:00
03eed88917 v7.0.24 2017-10-29 11:38:11 -07:00
60a7ad96b5 changes for v7.0.24 2017-10-29 11:37:31 -07:00
2d1d5d336a link to issues from changelog 2017-10-29 11:35:38 -07:00
e659759b2d upgrade wakatime-cli to v8.0.5 2017-10-29 11:32:03 -07:00
a290e5d86d v7.0.23 2017-09-14 17:54:06 -07:00
d5b922bb10 changes for 7.0.23 2017-09-14 17:53:51 -07:00
ec7b5e3530 add missing setting description 2017-09-14 17:51:22 -07:00
aa3f2e8af6 Merge pull request #76 from krishnaglick/master
Adding "Include" Functionality
2017-09-14 17:46:55 -07:00
f4e53cd682 Added 'include' functionality 2017-09-14 13:54:50 -04:00
aba72b0f1e v7.0.22 2017-06-08 00:21:33 -07:00
5b9d86a57d changes for v7.0.22 2017-06-08 00:21:15 -07:00
fa40874635 upgrade wakatime-cli to v8.0.3 2017-06-08 00:20:10 -07:00
6d4a4cf9eb link to FAQ from troubleshooting section 2017-05-26 07:40:14 -07:00
f628b8dd11 v7.0.21 2017-05-24 23:55:36 -07:00
f932ee9fc6 changes for v7.0.21 2017-05-24 23:55:17 -07:00
2f14009279 Upgrade wakatime-cli to v8.0.2 2017-05-24 23:53:28 -07:00
453d96bf9c use unicode arrow 2017-05-02 19:24:54 -07:00
9de153f156 use unicode arrow 2017-04-15 10:13:42 -07:00
dcc782338d v7.0.20 2017-04-10 19:04:00 -07:00
9d0dba988a remove sub-list install instructions 2017-04-10 19:03:05 -07:00
980 changed files with 1119 additions and 141701 deletions

View File

@ -13,3 +13,5 @@ Patches and Suggestions
- Jimmy Selgen Nielsen <jimmy.selgen@gmail.com>
- Patrik Kernstock <info@pkern.at>
- Krishna Glick <krishnaglick@gmail.com>
- Carlos Henrique Gandarez <gandarez@gmail.com>

View File

@ -3,6 +3,439 @@ History
-------
11.1.0 (2022-11-11)
++++++++++++++++++
- Support for api key vault cmd config
11.0.8 (2022-08-23)
++++++++++++++++++
- Bugfix to prevent using empty selection object.
`#116 <https://github.com/wakatime/sublime-wakatime/issues/116>`_
11.0.7 (2022-06-25)
++++++++++++++++++
- Check wakatime-cli versions in background thread.
`#115 <https://github.com/wakatime/sublime-wakatime/issues/115>`_
11.0.6 (2022-06-08)
++++++++++++++++++
- Fix call to log helper.
`#113 <https://github.com/wakatime/sublime-wakatime/issues/113>`_
11.0.5 (2022-04-29)
++++++++++++++++++
- Bugfix to not overwrite global urlopener in embedded Python.
`#110 <https://github.com/wakatime/sublime-wakatime/issues/110>`_
- Chmod wakatime-cli to be executable after updating on non-Windows platforms.
11.0.4 (2022-01-06)
++++++++++++++++++
- Copy wakatime-cli when symlink fails on Windows.
`vim-wakatime#122 <https://github.com/wakatime/vim-wakatime/issues/122>`_
- Fix lineno, cursorpos, and lines-in-file arguments.
11.0.3 (2021-12-31)
++++++++++++++++++
- Bugfix to not delete old wakatime-cli until finished downloading new version.
`#107 <https://github.com/wakatime/sublime-wakatime/issues/107>`_
11.0.2 (2021-11-17)
++++++++++++++++++
- Bugfix to encode extra heartbeats cursorpos as int not str when sending to wakatime-cli.
11.0.1 (2021-11-16)
++++++++++++++++++
- Bugfix for install script when using system Python3 and duplicat INI keys.
11.0.0 (2021-10-31)
++++++++++++++++++
- Use new Go wakatime-cli.
10.0.1 (2020-12-28)
++++++++++++++++++
- Improve readme subtitle.
10.0.0 (2020-12-28)
++++++++++++++++++
- Support for standalone wakatime-cli, disabled by default.
9.1.2 (2020-02-13)
++++++++++++++++++
- Upgrade wakatime-cli to v13.0.7.
- Split bundled pygments library for Python 2.7+.
- Upgrade pygments for py27+ to v2.5.2 development master.
- Force requests to use bundled ca cert from certifi by default.
- Upgrade bundled certifi to v2019.11.28.
9.1.1 (2020-02-11)
++++++++++++++++++
- Fix typo in python detection on Windows platform.
9.1.0 (2020-02-09)
++++++++++++++++++
- Detect python in Windows LocalAppData install locations.
- Upgrade wakatime-cli to v13.0.4.
- Bundle cryptography, pyopenssl, and ipaddress packages for improved SSL
support on Python2.
9.0.2 (2019-12-04)
++++++++++++++++++
- Upgrade wakatime-cli to v13.0.3.
- Support slashes in Mercurial and Git branch names.
`wakatime#199 <https://github.com/wakatime/wakatime/issues/199>`_
9.0.1 (2019-11-24)
++++++++++++++++++
- Upgrade wakatime-cli to v13.0.2.
- Filter dependencies longer than 200 characters.
- Close sqlite connection even when error raised.
`wakatime#196 <https://github.com/wakatime/wakatime/issues/196>`_
- Detect ColdFusion as root language instead of HTML.
- New arguments for reading and writing ini config file.
- Today argument shows categories when available.
- Prevent unnecessarily debug log when syncing offline heartbeats.
- Support for Python 3.7.
9.0.0 (2019-06-23)
++++++++++++++++++
- New optional config option hide_branch_names.
`wakatime#183 <https://github.com/wakatime/wakatime/issues/183>`_
8.7.0 (2019-05-29)
++++++++++++++++++
- Prevent creating user sublime-settings file when api key already exists in
common wakatime.cfg file.
`#98 <https://github.com/wakatime/sublime-wakatime/issues/98>`_
8.6.1 (2019-05-28)
++++++++++++++++++
- Fix parsing common wakatime.cfg file.
`#98 <https://github.com/wakatime/sublime-wakatime/issues/98>`_
8.6.0 (2019-05-27)
++++++++++++++++++
- Prevent prompting for api key when found from config file.
`#98 <https://github.com/wakatime/sublime-wakatime/issues/98>`_
8.5.0 (2019-05-10)
++++++++++++++++++
- Remove clock icon from status bar.
- Use wakatime-cli to fetch status bar coding time.
8.4.2 (2019-05-07)
++++++++++++++++++
- Upgrade wakatime-cli to v11.0.0.
- Rename argument --show-time-today to --today.
- New argument --show-time-today for printing Today's coding time.
8.4.1 (2019-05-01)
++++++++++++++++++
- Use api subdomain for fetching today's coding activity.
8.4.0 (2019-05-01)
++++++++++++++++++
- Show today's coding time in status bar.
8.3.6 (2019-04-30)
++++++++++++++++++
- Upgrade wakatime-cli to v10.8.4.
- Use wakatime fork of certifi package.
`#95 <https://github.com/wakatime/sublime-wakatime/issues/95>`_
8.3.5 (2019-04-30)
++++++++++++++++++
- Upgrade wakatime-cli to v10.8.3.
- Upgrade certifi to version 2019.03.09.
8.3.4 (2019-03-30)
++++++++++++++++++
- Upgrade wakatime-cli to v10.8.2.
- Detect go.mod files as Go language.
`jetbrains-wakatime#119 <https://github.com/wakatime/jetbrains-wakatime/issues/119>`_
- Detect C++ language from all C++ file extensions.
`vscode-wakatime#87 <https://github.com/wakatime/vscode-wakatime/issues/87>`_
- Add ssl_certs_file arg and config for custom ca bundles.
`wakatime#164 <https://github.com/wakatime/wakatime/issues/164>`_
- Fix bug causing random project names when hide project names enabled.
`vscode-wakatime#162 <https://github.com/wakatime/vscode-wakatime/issues/61>`_
- Add support for UNC network shares without drive letter mapped on Winows.
`wakatime#162 <https://github.com/wakatime/wakatime/issues/162>`_
8.3.3 (2018-12-19)
++++++++++++++++++
- Upgrade wakatime-cli to v10.6.1.
- Correctly parse include_only_with_project_file when set to false.
`wakatime#161 <https://github.com/wakatime/wakatime/issues/161>`_
- Support language argument for non-file entity types.
- Send 25 heartbeats per API request.
- New category "Writing Tests".
`wakatime#156 <https://github.com/wakatime/wakatime/issues/156>`_
- Fix bug caused by git config section without any submodule option defined.
`wakatime#152 <https://github.com/wakatime/wakatime/issues/152>`_
8.3.2 (2018-10-06)
++++++++++++++++++
- Send buffered heartbeats to API every 30 seconds.
8.3.1 (2018-10-05)
++++++++++++++++++
- Upgrade wakatime-cli to v10.4.1.
- Send 50 offline heartbeats to API per request with 1 second delay in between.
8.3.0 (2018-10-03)
++++++++++++++++++
- Upgrade wakatime-cli to v10.4.0.
- Support logging coding activity to remote network drive files on Windows
platform by detecting UNC path from drive letter.
`wakatime#72 <https://github.com/wakatime/wakatime/issues/72>`_
8.2.0 (2018-09-30)
++++++++++++++++++
- Prevent opening cmd window on Windows when running wakatime-cli.
`#91 <https://github.com/wakatime/sublime-wakatime/issues/91>`_
- Upgrade wakatime-cli to v10.3.0.
- Re-enable detecting projects from Subversion folder on Windows platform.
- Prevent opening cmd window on Windows when detecting project from Subversion.
- Run tests on Windows using Appveyor.
8.1.2 (2018-09-20)
++++++++++++++++++
- Upgrade wakatime-cli to v10.2.4.
- Default --sync-offline-activity to 100 instead of 5, so offline coding is
synced to dashboard faster.
- Batch heartbeats in groups of 10 per api request.
- New config hide_project_name and argument --hide-project-names for
obfuscating project names when sending coding activity to api.
- Fix mispelled Gosu language.
`wakatime#137 <https://github.com/wakatime/wakatime/issues/137>`_
- Remove metadata when hiding project or file names.
- New --local-file argument to be used when --entity is a remote file.
- New argument --sync-offline-activity for configuring the maximum offline
heartbeats to sync to the WakaTime API.
8.1.1 (2018-04-26)
++++++++++++++++++
- Upgrade wakatime-cli to v10.2.1.
- Force forward slash for file paths.
- New --category argument.
- New --exclude-unknown-project argument and corresponding config setting.
- Support for project detection from git worktree folders.
8.1.0 (2018-04-03)
++++++++++++++++++
- Prefer Python3 over Python2 when running wakatime-cli core.
- Improve detection of Python3 on Ubuntu 17.10 platforms.
8.0.8 (2018-03-15)
++++++++++++++++++
- Upgrade wakatime-cli to v10.1.3.
- Smarter C vs C++ vs Objective-C language detection.
8.0.7 (2018-03-15)
++++++++++++++++++
- Upgrade wakatime-cli to v10.1.2.
- Detect dependencies from Swift, Objective-C, TypeScript and JavaScript files.
- Categorize .mjs files as JavaScript.
`wakatime#121 <https://github.com/wakatime/wakatime/issues/121>`_
- Detect dependencies from Elm, Haskell, Haxe, Kotlin, Rust, and Scala files.
- Improved Matlab vs Objective-C language detection.
`wakatime#129 <https://github.com/wakatime/wakatime/issues/129>`_
8.0.6 (2018-01-04)
++++++++++++++++++
- Upgrade wakatime-cli to v10.1.0.
- Ability to only track folders containing a .wakatime-project file using new
include_only_with_project_file argument and config option.
8.0.5 (2017-11-24)
++++++++++++++++++
- Upgrade wakatime-cli to v10.0.5.
- Fix bug that caused heartbeats to be cached locally instead of sent to API.
8.0.4 (2017-11-23)
++++++++++++++++++
- Upgrade wakatime-cli to v10.0.4.
- Improve Java dependency detection.
- Skip null or missing heartbeats from extra heartbeats argument.
8.0.3 (2017-11-22)
++++++++++++++++++
- Upgrade wakatime-cli to v10.0.3.
- Support saving unicode heartbeats when working offline.
`wakatime#112 <https://github.com/wakatime/wakatime/issues/112>`_
8.0.2 (2017-11-15)
++++++++++++++++++
- Upgrade wakatime-cli to v10.0.2.
- Limit bulk syncing to 5 heartbeats per request.
`wakatime#109 <https://github.com/wakatime/wakatime/issues/109>`_
8.0.1 (2017-11-09)
++++++++++++++++++
- Upgrade wakatime-cli to v10.0.1.
- Parse array of results from bulk heartbeats endpoint, only saving heartbeats
to local offline cache when they were not accepted by the api.
8.0.0 (2017-11-08)
++++++++++++++++++
- Upgrade wakatime-cli to v10.0.0.
- Upload multiple heartbeats to bulk endpoint for improved network performance.
`wakatime#107 <https://github.com/wakatime/wakatime/issues/107>`_
7.0.26 (2017-11-07)
++++++++++++++++++
- Upgrade wakatime-cli to v9.0.1.
- Fix bug causing 401 response when hidefilenames is enabled.
`wakatime#106 <https://github.com/wakatime/wakatime/issues/106>`_
7.0.25 (2017-11-05)
++++++++++++++++++
- Ability to override python binary location in sublime-settings file.
`#78 <https://github.com/wakatime/sublime-wakatime/issues/78>`_
- Upgrade wakatime-cli to v9.0.0.
- Detect project and branch names from git submodules.
`wakatime#105 <https://github.com/wakatime/wakatime/issues/105>`_
7.0.24 (2017-10-29)
++++++++++++++++++
- Upgrade wakatime-cli to v8.0.5.
- Allow passing string arguments wrapped in extra quotes for plugins which
cannot properly escape spaces in arguments.
- Upgrade pytz to v2017.2.
- Upgrade requests to v2.18.4.
- Upgrade tzlocal to v1.4.
- Use WAKATIME_HOME env variable for offline and session caching.
`wakatime#102 <https://github.com/wakatime/wakatime/issues/102>`_
7.0.23 (2017-09-14)
++++++++++++++++++
- Add "include" setting to bypass ignored files.
`#89 <https://github.com/wakatime/sublime-wakatime/issues/89>`_
7.0.22 (2017-06-08)
++++++++++++++++++
- Upgrade wakatime-cli to v8.0.3.
- Improve Matlab language detection.
7.0.21 (2017-05-24)
++++++++++++++++++
- Upgrade wakatime-cli to v8.0.2.
- Only treat proxy string as NTLM proxy after unable to connect with HTTPS and
SOCKS proxy.
- Support running automated tests on Linux, OS X, and Windows.
- Ability to disable SSL cert verification.
`wakatime#90 <https://github.com/wakatime/wakatime/issues/90>`_
- Disable line count stats for files larger than 2MB to improve performance.
- Print error saying Python needs upgrading when requests can't be imported.
7.0.20 (2017-04-10)
++++++++++++++++++
- Fix install instructions formatting.
7.0.19 (2017-04-10)
++++++++++++++++++
@ -62,13 +495,15 @@ History
7.0.11 (2016-09-23)
++++++++++++++++++
- Handle UnicodeDecodeError when when logging. Related to #68.
- Handle UnicodeDecodeError when when logging.
`#68 <https://github.com/wakatime/sublime-wakatime/issues/68>`_
7.0.10 (2016-09-22)
++++++++++++++++++
- Handle UnicodeDecodeError when looking for python. Fixes #68.
- Handle UnicodeDecodeError when looking for python.
`#68 <https://github.com/wakatime/sublime-wakatime/issues/68>`_
- Upgrade wakatime-cli to v6.0.9.

View File

@ -1,7 +1,6 @@
BSD 3-Clause License
Copyright (c) 2014 by the respective authors (see AUTHORS file).
All rights reserved.
Copyright (c) 2014 Alan Hamlett.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

View File

@ -1,47 +1,38 @@
sublime-wakatime
================
# sublime-wakatime
Metrics, insights, and time tracking automatically generated from your programming activity.
[![Coding time tracker](https://wakatime.com/badge/github/wakatime/sublime-wakatime.svg)](https://wakatime.com/badge/github/wakatime/sublime-wakatime)
[WakaTime][wakatime] is an open source Sublime Text plugin for metrics, insights, and time tracking automatically generated from your programming activity.
Installation
------------
## Installation
1. Install [Package Control](https://packagecontrol.io/installation).
2. Using [Package Control](https://packagecontrol.io/docs/usage):
2. In Sublime, press `ctrl+shift+p`(Windows, Linux) or `cmd+shift+p`(OS X).
a) Inside Sublime, press `ctrl+shift+p`(Windows, Linux) or `cmd+shift+p`(OS X).
3. Type `install`, then press `enter` with `Package Control: Install Package` selected.
b) Type `install`, then press `enter` with `Package Control: Install Package` selected.
4. Type `wakatime`, then press `enter` with the `WakaTime` plugin selected.
c) Type `wakatime`, then press `enter` with the `WakaTime` plugin selected.
5. Enter your [api key](https://wakatime.com/settings#apikey), then press `enter`.
3. Enter your [api key](https://wakatime.com/settings#apikey), then press `enter`.
6. Use Sublime and your coding activity will be displayed on your [WakaTime dashboard](https://wakatime.com).
4. Use Sublime and your coding activity will be displayed on your [WakaTime dashboard](https://wakatime.com).
Screen Shots
------------
## Screen Shots
![Project Overview](https://wakatime.com/static/img/ScreenShots/Screen-Shot-2016-03-21.png)
Unresponsive Plugin Warning
---------------------------
## Unresponsive Plugin Warning
In Sublime Text 2, if you get a warning message:
A plugin (WakaTime) may be making Sublime Text unresponsive by taking too long (0.017332s) in its on_modified callback.
To fix this, go to `Preferences > Settings - User` then add the following setting:
To fix this, go to `Preferences Settings - User` then add the following setting:
`"detect_slow_plugins": false`
Troubleshooting
---------------
## Troubleshooting
First, turn on debug mode in your `WakaTime.sublime-settings` file.
@ -49,7 +40,13 @@ First, turn on debug mode in your `WakaTime.sublime-settings` file.
Add the line: `"debug": true`
Then, open your Sublime Console with `View -> Show Console` ( CTRL + \` ) to see the plugin executing the wakatime cli process when sending a heartbeat.
Then, open your Sublime Console with `View Show Console` ( CTRL + \` ) to see the plugin executing the wakatime cli process when sending a heartbeat.
Also, tail your `$HOME/.wakatime.log` file to debug wakatime cli problems.
For more general troubleshooting information, see [wakatime/wakatime#troubleshooting](https://github.com/wakatime/wakatime#troubleshooting).
The [How to Debug Plugins][how to debug] guide shows how to check when coding activity was last received from your editor using the [User Agents API][user agents api].
For more general troubleshooting info, see the [wakatime-cli Troubleshooting Section][wakatime-cli-help].
[wakatime]: https://wakatime.com/sublime-text
[wakatime-cli-help]: https://github.com/wakatime/wakatime#troubleshooting
[how to debug]: https://wakatime.com/faq#debug-plugins
[user agents api]: https://wakatime.com/developers#user_agents

File diff suppressed because it is too large Load Diff

View File

@ -3,7 +3,7 @@
// This settings file will be overwritten when upgrading.
{
// Your api key from https://wakatime.com/#apikey
// Your api key from https://wakatime.com/api-key
// Set this in your User specific WakaTime.sublime-settings file.
"api_key": "",
@ -17,13 +17,24 @@
// POSIX regular expressions will not be logged.
"ignore": ["^/tmp/", "^/etc/", "^/var/(?!www/).*", "COMMIT_EDITMSG$", "PULLREQ_EDITMSG$", "MERGE_MSG$", "TAG_EDITMSG$"],
// Status bar message. Set to false to hide status bar message.
// Defaults to true.
"status_bar_message": true,
// Include files; Files (including absolute paths) that match one of these
// POSIX regular expressions will bypass your ignore setting.
"include": [".*"],
// Status bar message format.
"status_bar_message_fmt": "WakaTime {status} %I:%M %p",
// Status bar for surfacing errors and displaying today's coding time. Set
// to false to hide. Defaults to true.
"status_bar_enabled": true,
// Show today's coding activity in WakaTime status bar item.
// Defaults to true.
"status_bar_coding_activity": true,
// Obfuscate file paths when sending to API. Your dashboard will no longer display coding activity per file.
"hidefilenames": false
"hidefilenames": false,
// Python binary location. Uses python from your PATH by default.
"python_binary": "",
// Use standalone compiled Python wakatime-cli (Will not work on ARM Macs)
"standalone": false
}

View File

@ -1,9 +0,0 @@
__title__ = 'wakatime'
__description__ = 'Common interface to the WakaTime api.'
__url__ = 'https://github.com/wakatime/wakatime'
__version_info__ = ('8', '0', '0')
__version__ = '.'.join(__version_info__)
__author__ = 'Alan Hamlett'
__author_email__ = 'alan@wakatime.com'
__license__ = 'BSD'
__copyright__ = 'Copyright 2017 Alan Hamlett'

View File

@ -1,17 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime
~~~~~~~~
Common interface to the WakaTime api.
http://wakatime.com
:copyright: (c) 2013 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
__all__ = ['main']
from .main import execute

View File

@ -1,234 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime.arguments
~~~~~~~~~~~~~~~~~~
Command-line arguments.
:copyright: (c) 2016 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
from __future__ import print_function
import os
import re
import time
import traceback
from .__about__ import __version__
from .configs import parseConfigFile
from .constants import AUTH_ERROR
from .packages import argparse
class FileAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
try:
if os.path.isfile(values):
values = os.path.realpath(values)
except: # pragma: nocover
pass
setattr(namespace, self.dest, values)
def parseArguments():
"""Parse command line arguments and configs from ~/.wakatime.cfg.
Command line arguments take precedence over config file settings.
Returns instances of ArgumentParser and SafeConfigParser.
"""
# define supported command line arguments
parser = argparse.ArgumentParser(
description='Common interface for the WakaTime api.')
parser.add_argument('--entity', dest='entity', metavar='FILE',
action=FileAction,
help='absolute path to file for the heartbeat; can also be a '+
'url, domain, or app when --entity-type is not file')
parser.add_argument('--file', dest='file', action=FileAction,
help=argparse.SUPPRESS)
parser.add_argument('--key', dest='key',
help='your wakatime api key; uses api_key from '+
'~/.wakatime.cfg by default')
parser.add_argument('--write', dest='is_write',
action='store_true',
help='when set, tells api this heartbeat was triggered from '+
'writing to a file')
parser.add_argument('--plugin', dest='plugin',
help='optional text editor plugin name and version '+
'for User-Agent header')
parser.add_argument('--time', dest='timestamp', metavar='time',
type=float,
help='optional floating-point unix epoch timestamp; '+
'uses current time by default')
parser.add_argument('--lineno', dest='lineno',
help='optional line number; current line being edited')
parser.add_argument('--cursorpos', dest='cursorpos',
help='optional cursor position in the current file')
parser.add_argument('--entity-type', dest='entity_type',
help='entity type for this heartbeat. can be one of "file", '+
'"domain", or "app"; defaults to file.')
parser.add_argument('--proxy', dest='proxy',
help='optional proxy configuration. Supports HTTPS '+
'and SOCKS proxies. For example: '+
'https://user:pass@host:port or '+
'socks5://user:pass@host:port or ' +
'domain\\user:pass')
parser.add_argument('--project', dest='project',
help='optional project name')
parser.add_argument('--alternate-project', dest='alternate_project',
help='optional alternate project name; auto-discovered project '+
'takes priority')
parser.add_argument('--alternate-language', dest='alternate_language',
help=argparse.SUPPRESS)
parser.add_argument('--language', dest='language',
help='optional language name; if valid, takes priority over '+
'auto-detected language')
parser.add_argument('--hostname', dest='hostname', help='hostname of '+
'current machine.')
parser.add_argument('--disableoffline', dest='offline',
action='store_false',
help='disables offline time logging instead of queuing logged time')
parser.add_argument('--hidefilenames', dest='hidefilenames',
action='store_true',
help='obfuscate file names; will not send file names to api')
parser.add_argument('--exclude', dest='exclude', action='append',
help='filename patterns to exclude from logging; POSIX regex '+
'syntax; can be used more than once')
parser.add_argument('--include', dest='include', action='append',
help='filename patterns to log; when used in combination with '+
'--exclude, files matching include will still be logged; '+
'POSIX regex syntax; can be used more than once')
parser.add_argument('--ignore', dest='ignore', action='append',
help=argparse.SUPPRESS)
parser.add_argument('--extra-heartbeats', dest='extra_heartbeats',
action='store_true',
help='reads extra heartbeats from STDIN as a JSON array until EOF')
parser.add_argument('--logfile', dest='logfile',
help='defaults to ~/.wakatime.log')
parser.add_argument('--apiurl', dest='api_url',
help='heartbeats api url; for debugging with a local server')
parser.add_argument('--timeout', dest='timeout', type=int,
help='number of seconds to wait when sending heartbeats to api; '+
'defaults to 60 seconds')
parser.add_argument('--config', dest='config',
help='defaults to ~/.wakatime.cfg')
parser.add_argument('--verbose', dest='verbose', action='store_true',
help='turns on debug messages in log file')
parser.add_argument('--version', action='version', version=__version__)
# parse command line arguments
args = parser.parse_args()
# use current unix epoch timestamp by default
if not args.timestamp:
args.timestamp = time.time()
# parse ~/.wakatime.cfg file
configs = parseConfigFile(args.config)
# update args from configs
if not args.hostname:
if configs.has_option('settings', 'hostname'):
args.hostname = configs.get('settings', 'hostname')
if not args.key:
default_key = None
if configs.has_option('settings', 'api_key'):
default_key = configs.get('settings', 'api_key')
elif configs.has_option('settings', 'apikey'):
default_key = configs.get('settings', 'apikey')
if default_key:
args.key = default_key
else:
try:
parser.error('Missing api key. Find your api key from wakatime.com/settings.')
except SystemExit:
raise SystemExit(AUTH_ERROR)
is_valid = not not re.match(r'^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$', args.key, re.I)
if not is_valid:
try:
parser.error('Invalid api key. Find your api key from wakatime.com/settings.')
except SystemExit:
raise SystemExit(AUTH_ERROR)
if not args.entity:
if args.file:
args.entity = args.file
else:
parser.error('argument --entity is required')
if not args.language and args.alternate_language:
args.language = args.alternate_language
if not args.exclude:
args.exclude = []
if configs.has_option('settings', 'ignore'):
try:
for pattern in configs.get('settings', 'ignore').split("\n"):
if pattern.strip() != '':
args.exclude.append(pattern)
except TypeError: # pragma: nocover
pass
if configs.has_option('settings', 'exclude'):
try:
for pattern in configs.get('settings', 'exclude').split("\n"):
if pattern.strip() != '':
args.exclude.append(pattern)
except TypeError: # pragma: nocover
pass
if not args.include:
args.include = []
if configs.has_option('settings', 'include'):
try:
for pattern in configs.get('settings', 'include').split("\n"):
if pattern.strip() != '':
args.include.append(pattern)
except TypeError: # pragma: nocover
pass
if args.hidefilenames:
args.hidefilenames = ['.*']
else:
args.hidefilenames = []
if configs.has_option('settings', 'hidefilenames'):
option = configs.get('settings', 'hidefilenames')
if option.strip().lower() == 'true':
args.hidefilenames = ['.*']
elif option.strip().lower() != 'false':
for pattern in option.split("\n"):
if pattern.strip() != '':
args.hidefilenames.append(pattern)
if args.offline and configs.has_option('settings', 'offline'):
args.offline = configs.getboolean('settings', 'offline')
if not args.proxy and configs.has_option('settings', 'proxy'):
args.proxy = configs.get('settings', 'proxy')
if args.proxy:
pattern = r'^((https?|socks5)://)?([^:@]+(:([^:@])+)?@)?[^:]+(:\d+)?$'
if '\\' in args.proxy:
pattern = r'^.*\\.+$'
is_valid = not not re.match(pattern, args.proxy, re.I)
if not is_valid:
parser.error('Invalid proxy. Must be in format ' +
'https://user:pass@host:port or ' +
'socks5://user:pass@host:port or ' +
'domain\\user:pass.')
if not args.verbose and configs.has_option('settings', 'verbose'):
args.verbose = configs.getboolean('settings', 'verbose')
if not args.verbose and configs.has_option('settings', 'debug'):
args.verbose = configs.getboolean('settings', 'debug')
if not args.logfile and configs.has_option('settings', 'logfile'):
args.logfile = configs.get('settings', 'logfile')
if not args.logfile and os.environ.get('WAKATIME_HOME'):
home = os.environ.get('WAKATIME_HOME')
args.logfile = os.path.join(os.path.expanduser(home), '.wakatime.log')
if not args.api_url and configs.has_option('settings', 'api_url'):
args.api_url = configs.get('settings', 'api_url')
if not args.timeout and configs.has_option('settings', 'timeout'):
try:
args.timeout = int(configs.get('settings', 'timeout'))
except ValueError:
print(traceback.format_exc())
return args, configs

View File

@ -1,35 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime.cli
~~~~~~~~~~~~
Command-line entry point.
:copyright: (c) 2013 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
import os
import sys
# get path to local wakatime package
package_folder = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# add local wakatime package to sys.path
sys.path.insert(0, package_folder)
# import local wakatime package
try:
import wakatime
except (TypeError, ImportError):
# on Windows, non-ASCII characters in import path can be fixed using
# the script path from sys.argv[0].
# More info at https://github.com/wakatime/wakatime/issues/32
package_folder = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
sys.path.insert(0, package_folder)
import wakatime
if __name__ == '__main__':
sys.exit(wakatime.execute(sys.argv[1:]))

View File

@ -1,93 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime.compat
~~~~~~~~~~~~~~~
For working with Python2 and Python3.
:copyright: (c) 2014 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
import codecs
import sys
is_py2 = (sys.version_info[0] == 2)
is_py3 = (sys.version_info[0] == 3)
if is_py2: # pragma: nocover
def u(text):
if text is None:
return None
try:
return text.decode('utf-8')
except:
try:
return text.decode(sys.getdefaultencoding())
except:
try:
return unicode(text)
except:
return text.decode('utf-8', 'replace')
open = codecs.open
basestring = basestring
elif is_py3: # pragma: nocover
def u(text):
if text is None:
return None
if isinstance(text, bytes):
try:
return text.decode('utf-8')
except:
try:
return text.decode(sys.getdefaultencoding())
except:
pass
try:
return str(text)
except:
return text.decode('utf-8', 'replace')
open = open
basestring = (str, bytes)
try:
from importlib import import_module
except ImportError: # pragma: nocover
def _resolve_name(name, package, level):
"""Return the absolute name of the module to be imported."""
if not hasattr(package, 'rindex'):
raise ValueError("'package' not set to a string")
dot = len(package)
for x in xrange(level, 1, -1):
try:
dot = package.rindex('.', 0, dot)
except ValueError:
raise ValueError("attempted relative import beyond top-level "
"package")
return "%s.%s" % (package[:dot], name)
def import_module(name, package=None):
"""Import a module.
The 'package' argument is required when performing a relative import.
It specifies the package to use as the anchor point from which to
resolve the relative import to an absolute import.
"""
if name.startswith('.'):
if not package:
raise TypeError("relative imports require the 'package' "
+ "argument")
level = 0
for character in name:
if character != '.':
break
level += 1
name = _resolve_name(name[level:], package, level)
__import__(name)
return sys.modules[name]

View File

@ -1,53 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime.configs
~~~~~~~~~~~~~~~~
Config file parser.
:copyright: (c) 2016 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
from __future__ import print_function
import os
import traceback
from .compat import open
from .constants import CONFIG_FILE_PARSE_ERROR
try:
import configparser
except ImportError:
from .packages import configparser
def parseConfigFile(configFile=None):
"""Returns a configparser.SafeConfigParser instance with configs
read from the config file. Default location of the config file is
at ~/.wakatime.cfg.
"""
# get config file location from ENV
home = os.environ.get('WAKATIME_HOME')
if not configFile and home:
configFile = os.path.join(os.path.expanduser(home), '.wakatime.cfg')
# use default config file location
if not configFile:
configFile = os.path.join(os.path.expanduser('~'), '.wakatime.cfg')
configs = configparser.ConfigParser(delimiters=('='), strict=False)
try:
with open(configFile, 'r', encoding='utf-8') as fh:
try:
configs.read_file(fh)
except configparser.Error:
print(traceback.format_exc())
raise SystemExit(CONFIG_FILE_PARSE_ERROR)
except IOError:
pass
return configs

View File

@ -1,40 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime.constants
~~~~~~~~~~~~~~~~~~
Constant variable definitions.
:copyright: (c) 2016 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
""" Success
Exit code used when a heartbeat was sent successfully.
"""
SUCCESS = 0
""" Api Error
Exit code used when the WakaTime API returned an error.
"""
API_ERROR = 102
""" Config File Parse Error
Exit code used when the ~/.wakatime.cfg config file could not be parsed.
"""
CONFIG_FILE_PARSE_ERROR = 103
""" Auth Error
Exit code used when our api key is invalid.
"""
AUTH_ERROR = 104
""" Unknown Error
Exit code used when there was an unhandled exception.
"""
UNKNOWN_ERROR = 105
""" Malformed Heartbeat Error
Exit code used when the JSON input from `--extra-heartbeats` is malformed.
"""
MALFORMED_HEARTBEAT_ERROR = 106

View File

@ -1,129 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime.dependencies
~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from a source code file.
:copyright: (c) 2014 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
import logging
import re
import sys
from ..compat import u, open, import_module
from ..exceptions import NotYetImplemented
log = logging.getLogger('WakaTime')
class TokenParser(object):
"""The base class for all dependency parsers. To add support for your
language, inherit from this class and implement the :meth:`parse` method
to return a list of dependency strings.
"""
exclude = []
def __init__(self, source_file, lexer=None):
self._tokens = None
self.dependencies = []
self.source_file = source_file
self.lexer = lexer
self.exclude = [re.compile(x, re.IGNORECASE) for x in self.exclude]
@property
def tokens(self):
if self._tokens is None:
self._tokens = self._extract_tokens()
return self._tokens
def parse(self, tokens=[]):
""" Should return a list of dependencies.
"""
raise NotYetImplemented()
def append(self, dep, truncate=False, separator=None, truncate_to=None,
strip_whitespace=True):
self._save_dependency(
dep,
truncate=truncate,
truncate_to=truncate_to,
separator=separator,
strip_whitespace=strip_whitespace,
)
def partial(self, token):
return u(token).split('.')[-1]
def _extract_tokens(self):
if self.lexer:
try:
with open(self.source_file, 'r', encoding='utf-8') as fh:
return self.lexer.get_tokens_unprocessed(fh.read(512000))
except:
pass
try:
with open(self.source_file, 'r', encoding=sys.getfilesystemencoding()) as fh:
return self.lexer.get_tokens_unprocessed(fh.read(512000)) # pragma: nocover
except:
pass
return []
def _save_dependency(self, dep, truncate=False, separator=None,
truncate_to=None, strip_whitespace=True):
if truncate:
if separator is None:
separator = u('.')
separator = u(separator)
dep = dep.split(separator)
if truncate_to is None or truncate_to < 1:
truncate_to = 1
if truncate_to > len(dep):
truncate_to = len(dep)
dep = dep[0] if len(dep) == 1 else separator.join(dep[:truncate_to])
if strip_whitespace:
dep = dep.strip()
if dep and (not separator or not dep.startswith(separator)):
should_exclude = False
for compiled in self.exclude:
if compiled.search(dep):
should_exclude = True
break
if not should_exclude:
self.dependencies.append(dep)
class DependencyParser(object):
source_file = None
lexer = None
parser = None
def __init__(self, source_file, lexer):
self.source_file = source_file
self.lexer = lexer
if self.lexer:
module_name = self.lexer.__module__.rsplit('.', 1)[-1]
class_name = self.lexer.__class__.__name__.replace('Lexer', 'Parser', 1)
else:
module_name = 'unknown'
class_name = 'UnknownParser'
try:
module = import_module('.%s' % module_name, package=__package__)
try:
self.parser = getattr(module, class_name)
except AttributeError:
log.debug('Parsing dependencies not supported for {0}.{1}'.format(module_name, class_name))
except ImportError:
log.debug('Parsing dependencies not supported for {0}.{1}'.format(module_name, class_name))
def parse(self):
if self.parser:
plugin = self.parser(self.source_file, lexer=self.lexer)
dependencies = plugin.parse()
return list(set(dependencies))
return []

View File

@ -1,51 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime.languages.c_cpp
~~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from C++ code.
:copyright: (c) 2014 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
from . import TokenParser
class CParser(TokenParser):
exclude = [
r'^stdio\.h$',
r'^stdlib\.h$',
r'^string\.h$',
r'^time\.h$',
]
state = None
def parse(self):
for index, token, content in self.tokens:
self._process_token(token, content)
return self.dependencies
def _process_token(self, token, content):
if self.partial(token) == 'Preproc' or self.partial(token) == 'PreprocFile':
self._process_preproc(token, content)
else:
self._process_other(token, content)
def _process_preproc(self, token, content):
if self.state == 'include':
if content != '\n' and content != '#':
content = content.strip().strip('"').strip('<').strip('>').strip()
self.append(content, truncate=True, separator='/')
self.state = None
elif content.strip().startswith('include'):
self.state = 'include'
else:
self.state = None
def _process_other(self, token, content):
pass
class CppParser(CParser):
pass

View File

@ -1,64 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime.languages.data
~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from data files.
:copyright: (c) 2014 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
import os
from . import TokenParser
from ..compat import u
FILES = {
'bower.json': {'exact': True, 'dependency': 'bower'},
'component.json': {'exact': True, 'dependency': 'bower'},
'package.json': {'exact': True, 'dependency': 'npm'},
}
class JsonParser(TokenParser):
state = None
level = 0
def parse(self):
self._process_file_name(os.path.basename(self.source_file))
for index, token, content in self.tokens:
self._process_token(token, content)
return self.dependencies
def _process_file_name(self, file_name):
for key, value in FILES.items():
found = (key == file_name) if value.get('exact') else (key.lower() in file_name.lower())
if found:
self.append(value['dependency'])
def _process_token(self, token, content):
if u(token) == 'Token.Name.Tag':
self._process_tag(token, content)
elif u(token) == 'Token.Literal.String.Single' or u(token) == 'Token.Literal.String.Double':
self._process_literal_string(token, content)
elif u(token) == 'Token.Punctuation':
self._process_punctuation(token, content)
def _process_tag(self, token, content):
if content.strip('"').strip("'") == 'dependencies' or content.strip('"').strip("'") == 'devDependencies':
self.state = 'dependencies'
elif self.state == 'dependencies' and self.level == 2:
self.append(content.strip('"').strip("'"))
def _process_literal_string(self, token, content):
pass
def _process_punctuation(self, token, content):
if content == '{':
self.level += 1
elif content == '}':
self.level -= 1
if self.state is not None and self.level <= 1:
self.state = None

View File

@ -1,64 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime.languages.dotnet
~~~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from .NET code.
:copyright: (c) 2014 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
from . import TokenParser
from ..compat import u
class CSharpParser(TokenParser):
exclude = [
r'^system$',
r'^microsoft$',
]
state = None
buffer = u('')
def parse(self):
for index, token, content in self.tokens:
self._process_token(token, content)
return self.dependencies
def _process_token(self, token, content):
if self.partial(token) == 'Keyword':
self._process_keyword(token, content)
if self.partial(token) == 'Namespace' or self.partial(token) == 'Name':
self._process_namespace(token, content)
elif self.partial(token) == 'Punctuation':
self._process_punctuation(token, content)
else:
self._process_other(token, content)
def _process_keyword(self, token, content):
if content == 'using':
self.state = 'import'
self.buffer = u('')
def _process_namespace(self, token, content):
if self.state == 'import':
if u(content) != u('import') and u(content) != u('package') and u(content) != u('namespace') and u(content) != u('static'):
if u(content) == u(';'): # pragma: nocover
self._process_punctuation(token, content)
else:
self.buffer += u(content)
def _process_punctuation(self, token, content):
if self.state == 'import':
if u(content) == u(';'):
self.append(self.buffer, truncate=True)
self.buffer = u('')
self.state = None
elif u(content) == u('='):
self.buffer = u('')
else:
self.buffer += u(content)
def _process_other(self, token, content):
pass

View File

@ -1,77 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime.languages.go
~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from Go code.
:copyright: (c) 2016 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
from . import TokenParser
class GoParser(TokenParser):
state = None
parens = 0
aliases = 0
exclude = [
r'^"fmt"$',
]
def parse(self):
for index, token, content in self.tokens:
self._process_token(token, content)
return self.dependencies
def _process_token(self, token, content):
if self.partial(token) == 'Namespace':
self._process_namespace(token, content)
elif self.partial(token) == 'Punctuation':
self._process_punctuation(token, content)
elif self.partial(token) == 'String':
self._process_string(token, content)
elif self.partial(token) == 'Text':
self._process_text(token, content)
elif self.partial(token) == 'Other':
self._process_other(token, content)
else:
self._process_misc(token, content)
def _process_namespace(self, token, content):
self.state = content
self.parens = 0
self.aliases = 0
def _process_string(self, token, content):
if self.state == 'import':
self.append(content, truncate=False)
def _process_punctuation(self, token, content):
if content == '(':
self.parens += 1
elif content == ')':
self.parens -= 1
elif content == '.':
self.aliases += 1
else:
self.state = None
def _process_text(self, token, content):
if self.state == 'import':
if content == "\n" and self.parens <= 0:
self.state = None
self.parens = 0
self.aliases = 0
else:
self.state = None
def _process_other(self, token, content):
if self.state == 'import':
self.aliases += 1
else:
self.state = None
def _process_misc(self, token, content):
self.state = None

View File

@ -1,96 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime.languages.java
~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from Java code.
:copyright: (c) 2014 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
from . import TokenParser
from ..compat import u
class JavaParser(TokenParser):
exclude = [
r'^java\.',
r'^javax\.',
r'^import$',
r'^package$',
r'^namespace$',
r'^static$',
]
state = None
buffer = u('')
def parse(self):
for index, token, content in self.tokens:
self._process_token(token, content)
return self.dependencies
def _process_token(self, token, content):
if self.partial(token) == 'Namespace':
self._process_namespace(token, content)
if self.partial(token) == 'Name':
self._process_name(token, content)
elif self.partial(token) == 'Attribute':
self._process_attribute(token, content)
elif self.partial(token) == 'Operator':
self._process_operator(token, content)
else:
self._process_other(token, content)
def _process_namespace(self, token, content):
if u(content) == u('import'):
self.state = 'import'
elif self.state == 'import':
keywords = [
u('package'),
u('namespace'),
u('static'),
]
if u(content) in keywords:
return
self.buffer = u('{0}{1}').format(self.buffer, u(content))
elif self.state == 'import-finished':
content = content.split(u('.'))
if len(content) == 1:
self.append(content[0])
elif len(content) > 1:
if len(content[0]) == 3:
content = content[1:]
if content[-1] == u('*'):
content = content[:len(content) - 1]
if len(content) == 1:
self.append(content[0])
elif len(content) > 1:
self.append(u('.').join(content[:2]))
self.state = None
def _process_name(self, token, content):
if self.state == 'import':
self.buffer = u('{0}{1}').format(self.buffer, u(content))
def _process_attribute(self, token, content):
if self.state == 'import':
self.buffer = u('{0}{1}').format(self.buffer, u(content))
def _process_operator(self, token, content):
if u(content) == u(';'):
self.state = 'import-finished'
self._process_namespace(token, self.buffer)
self.state = None
self.buffer = u('')
elif self.state == 'import':
self.buffer = u('{0}{1}').format(self.buffer, u(content))
def _process_other(self, token, content):
pass

View File

@ -1,85 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime.languages.php
~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from PHP code.
:copyright: (c) 2014 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
from . import TokenParser
from ..compat import u
class PhpParser(TokenParser):
state = None
parens = 0
def parse(self):
for index, token, content in self.tokens:
self._process_token(token, content)
return self.dependencies
def _process_token(self, token, content):
if self.partial(token) == 'Keyword':
self._process_keyword(token, content)
elif u(token) == 'Token.Literal.String.Single' or u(token) == 'Token.Literal.String.Double':
self._process_literal_string(token, content)
elif u(token) == 'Token.Name.Other':
self._process_name(token, content)
elif u(token) == 'Token.Name.Function':
self._process_function(token, content)
elif self.partial(token) == 'Punctuation':
self._process_punctuation(token, content)
elif self.partial(token) == 'Text':
self._process_text(token, content)
else:
self._process_other(token, content)
def _process_name(self, token, content):
if self.state == 'use':
self.append(content, truncate=True, separator=u("\\"))
def _process_function(self, token, content):
if self.state == 'use function':
self.append(content, truncate=True, separator=u("\\"))
self.state = 'use'
def _process_keyword(self, token, content):
if content == 'include' or content == 'include_once' or content == 'require' or content == 'require_once':
self.state = 'include'
elif content == 'use':
self.state = 'use'
elif content == 'as':
self.state = 'as'
elif self.state == 'use' and content == 'function':
self.state = 'use function'
else:
self.state = None
def _process_literal_string(self, token, content):
if self.state == 'include':
if content != '"' and content != "'":
content = content.strip()
if u(token) == 'Token.Literal.String.Double':
content = u("'{0}'").format(content)
self.append(content)
self.state = None
def _process_punctuation(self, token, content):
if content == '(':
self.parens += 1
elif content == ')':
self.parens -= 1
elif (self.state == 'use' or self.state == 'as') and content == ',':
self.state = 'use'
else:
self.state = None
def _process_text(self, token, content):
pass
def _process_other(self, token, content):
self.state = None

View File

@ -1,83 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime.languages.python
~~~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from Python code.
:copyright: (c) 2014 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
from . import TokenParser
class PythonParser(TokenParser):
state = None
parens = 0
nonpackage = False
exclude = [
r'^os$',
r'^sys$',
r'^sys\.',
r'^__future__$',
]
def parse(self):
for index, token, content in self.tokens:
self._process_token(token, content)
return self.dependencies
def _process_token(self, token, content):
if self.partial(token) == 'Namespace':
self._process_namespace(token, content)
elif self.partial(token) == 'Operator':
self._process_operator(token, content)
elif self.partial(token) == 'Punctuation':
self._process_punctuation(token, content)
elif self.partial(token) == 'Text':
self._process_text(token, content)
else:
self._process_other(token, content)
def _process_namespace(self, token, content):
if self.state is None:
self.state = content
else:
if content == 'as':
self.nonpackage = True
else:
self._process_import(token, content)
def _process_operator(self, token, content):
pass
def _process_punctuation(self, token, content):
if content == '(':
self.parens += 1
elif content == ')':
self.parens -= 1
self.nonpackage = False
def _process_text(self, token, content):
if self.state is not None:
if content == "\n" and self.parens == 0:
self.state = None
self.nonpackage = False
def _process_other(self, token, content):
pass
def _process_import(self, token, content):
if not self.nonpackage:
if self.state == 'from':
self.append(content, truncate=True, truncate_to=1)
self.state = 'from-2'
elif self.state == 'import':
self.append(content, truncate=True, truncate_to=1)
self.state = 'import-2'
elif self.state == 'import-2':
self.append(content, truncate=True, truncate_to=1)
else:
self.state = None
self.nonpackage = False

View File

@ -1,203 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime.languages.templates
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from Templates.
:copyright: (c) 2014 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
from . import TokenParser
from ..compat import u
""" If these keywords are found in the source file, treat them as a dependency.
Must be lower-case strings.
"""
KEYWORDS = [
'_',
'$',
'angular',
'assert', # probably mocha
'backbone',
'batman',
'c3',
'can',
'casper',
'chai',
'chaplin',
'd3',
'define', # probably require
'describe', # mocha or jasmine
'eco',
'ember',
'espresso',
'expect', # probably jasmine
'exports', # probably npm
'express',
'gulp',
'handlebars',
'highcharts',
'jasmine',
'jquery',
'jstz',
'ko', # probably knockout
'm', # probably mithril
'marionette',
'meteor',
'moment',
'monitorio',
'mustache',
'phantom',
'pickadate',
'pikaday',
'qunit',
'react',
'reactive',
'require', # probably the commonjs spec
'ripple',
'rivets',
'socketio',
'spine',
'thorax',
'underscore',
'vue',
'way',
'zombie',
]
class HtmlDjangoParser(TokenParser):
tags = []
opening_tag = False
getting_attrs = False
current_attr = None
current_attr_value = None
def parse(self):
for index, token, content in self.tokens:
self._process_token(token, content)
return self.dependencies
def _process_token(self, token, content):
if u(token) == 'Token.Punctuation':
self._process_punctuation(token, content)
elif u(token) == 'Token.Name.Tag':
self._process_tag(token, content)
elif u(token) == 'Token.Literal.String':
self._process_string(token, content)
elif u(token) == 'Token.Name.Attribute':
self._process_attribute(token, content)
@property
def current_tag(self):
return None if len(self.tags) == 0 else self.tags[0]
def _process_punctuation(self, token, content):
if content.startswith('</') or content.startswith('/'):
try:
self.tags.pop(0)
except IndexError:
# ignore errors from malformed markup
pass
self.opening_tag = False
self.getting_attrs = False
elif content.startswith('<'):
self.opening_tag = True
elif content.startswith('>'):
self.opening_tag = False
self.getting_attrs = False
def _process_tag(self, token, content):
if self.opening_tag:
self.tags.insert(0, content.replace('<', '', 1).strip().lower())
self.getting_attrs = True
self.current_attr = None
def _process_attribute(self, token, content):
if self.getting_attrs:
self.current_attr = content.lower().strip('=')
self.current_attr_value = None
def _process_string(self, token, content):
if self.getting_attrs and self.current_attr is not None:
if content.endswith('"') or content.endswith("'"):
if self.current_attr_value is not None:
self.current_attr_value += content
if self.current_tag == 'script' and self.current_attr == 'src':
self.append(self.current_attr_value)
self.current_attr = None
self.current_attr_value = None
else:
if len(content) == 1:
self.current_attr_value = content
else:
if self.current_tag == 'script' and self.current_attr == 'src':
self.append(content)
self.current_attr = None
self.current_attr_value = None
elif content.startswith('"') or content.startswith("'"):
if self.current_attr_value is None:
self.current_attr_value = content
class VelocityHtmlParser(HtmlDjangoParser):
pass
class MyghtyHtmlParser(HtmlDjangoParser):
pass
class MasonParser(HtmlDjangoParser):
pass
class MakoHtmlParser(HtmlDjangoParser):
pass
class CheetahHtmlParser(HtmlDjangoParser):
pass
class HtmlGenshiParser(HtmlDjangoParser):
pass
class RhtmlParser(HtmlDjangoParser):
pass
class HtmlPhpParser(HtmlDjangoParser):
pass
class HtmlSmartyParser(HtmlDjangoParser):
pass
class EvoqueHtmlParser(HtmlDjangoParser):
pass
class ColdfusionHtmlParser(HtmlDjangoParser):
pass
class LassoHtmlParser(HtmlDjangoParser):
pass
class HandlebarsHtmlParser(HtmlDjangoParser):
pass
class YamlJinjaParser(HtmlDjangoParser):
pass
class TwigHtmlParser(HtmlDjangoParser):
pass

View File

@ -1,33 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime.languages.unknown
~~~~~~~~~~~~~~~~~~~~~~~~~~
Parse dependencies from files of unknown language.
:copyright: (c) 2014 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
import os
from . import TokenParser
FILES = {
'bower': {'exact': False, 'dependency': 'bower'},
'grunt': {'exact': False, 'dependency': 'grunt'},
}
class UnknownParser(TokenParser):
def parse(self):
self._process_file_name(os.path.basename(self.source_file))
return self.dependencies
def _process_file_name(self, file_name):
for key, value in FILES.items():
found = (key == file_name) if value.get('exact') else (key.lower() in file_name.lower())
if found:
self.append(value['dependency'])

View File

@ -1,14 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime.exceptions
~~~~~~~~~~~~~~~~~~~
Custom exceptions.
:copyright: (c) 2015 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
class NotYetImplemented(Exception):
"""This method needs to be implemented."""

View File

@ -1,18 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime.language_priorities
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Overwrite pygments Lexer.priority attribute for specific languages.
:copyright: (c) 2017 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
LANGUAGES = {
'typescript': 0.01,
'perl': 0.01,
'perl6': 0.01,
'f#': 0.01,
}

View File

@ -1,81 +0,0 @@
{
"actionscript": "ActionScript",
"apacheconf": "ApacheConf",
"applescript": "AppleScript",
"asp": "ASP",
"assembly": "Assembly",
"awk": "Awk",
"bash": "Bash",
"basic": "Basic",
"brightscript": "BrightScript",
"c": "C",
"c#": "C#",
"c++": "C++",
"clojure": "Clojure",
"cocoa": "Cocoa",
"coffeescript": "CoffeeScript",
"coldfusion": "ColdFusion",
"common lisp": "Common Lisp",
"cshtml": "CSHTML",
"css": "CSS",
"dart": "Dart",
"delphi": "Delphi",
"elixir": "Elixir",
"elm": "Elm",
"emacs lisp": "Emacs Lisp",
"erlang": "Erlang",
"f#": "F#",
"fortran": "Fortran",
"go": "Go",
"gous": "Gosu",
"groovy": "Groovy",
"haml": "Haml",
"haskell": "Haskell",
"haxe": "Haxe",
"html": "HTML",
"ini": "INI",
"jade": "Jade",
"java": "Java",
"javascript": "JavaScript",
"json": "JSON",
"jsx": "JSX",
"kotlin": "Kotlin",
"less": "LESS",
"lua": "Lua",
"markdown": "Markdown",
"matlab": "Matlab",
"mustache": "Mustache",
"objective-c": "Objective-C",
"objective-c++": "Objective-C++",
"objective-j": "Objective-J",
"ocaml": "OCaml",
"perl": "Perl",
"php": "PHP",
"powershell": "PowerShell",
"prolog": "Prolog",
"puppet": "Puppet",
"python": "Python",
"r": "R",
"restructuredtext": "reStructuredText",
"ruby": "Ruby",
"rust": "Rust",
"sass": "Sass",
"scala": "Scala",
"scheme": "Scheme",
"scss": "SCSS",
"shell": "Shell",
"slim": "Slim",
"smalltalk": "Smalltalk",
"sql": "SQL",
"swift": "Swift",
"text": "Text",
"turing": "Turing",
"twig": "Twig",
"typescript": "TypeScript",
"typoscript": "TypoScript",
"vb.net": "VB.net",
"viml": "VimL",
"xaml": "XAML",
"xml": "XML",
"yaml": "YAML"
}

View File

@ -1,531 +0,0 @@
{
"a2ps": null,
"a65": "Assembly",
"aap": null,
"abap": null,
"abaqus": null,
"abc": null,
"abel": null,
"acedb": null,
"ada": null,
"aflex": null,
"ahdl": null,
"alsaconf": null,
"amiga": null,
"aml": null,
"ampl": null,
"ant": null,
"antlr": null,
"apache": null,
"apachestyle": null,
"arch": null,
"art": null,
"asm": "Assembly",
"asm68k": "Assembly",
"asmh8300": "Assembly",
"asn": null,
"aspperl": null,
"aspvbs": null,
"asterisk": null,
"asteriskvm": null,
"atlas": null,
"autohotkey": null,
"autoit": null,
"automake": null,
"ave": null,
"awk": null,
"ayacc": null,
"b": null,
"baan": null,
"basic": "Basic",
"bc": null,
"bdf": null,
"bib": null,
"bindzone": null,
"blank": null,
"bst": null,
"btm": null,
"bzr": null,
"c": "C",
"cabal": null,
"calendar": null,
"catalog": null,
"cdl": null,
"cdrdaoconf": null,
"cdrtoc": null,
"cf": null,
"cfg": null,
"ch": null,
"chaiscript": null,
"change": null,
"changelog": null,
"chaskell": null,
"cheetah": null,
"chill": null,
"chordpro": null,
"cl": null,
"clean": null,
"clipper": null,
"cmake": null,
"cmusrc": null,
"cobol": null,
"coco": null,
"conaryrecipe": null,
"conf": null,
"config": null,
"context": null,
"cpp": "C++",
"crm": null,
"crontab": "Crontab",
"cs": "C#",
"csc": null,
"csh": null,
"csp": null,
"css": null,
"cterm": null,
"ctrlh": null,
"cucumber": null,
"cuda": null,
"cupl": null,
"cuplsim": null,
"cvs": null,
"cvsrc": null,
"cweb": null,
"cynlib": null,
"cynpp": null,
"d": null,
"datascript": null,
"dcd": null,
"dcl": null,
"debchangelog": null,
"debcontrol": null,
"debsources": null,
"def": null,
"denyhosts": null,
"desc": null,
"desktop": null,
"dictconf": null,
"dictdconf": null,
"diff": null,
"dircolors": null,
"diva": null,
"django": null,
"dns": null,
"docbk": null,
"docbksgml": null,
"docbkxml": null,
"dosbatch": null,
"dosini": null,
"dot": null,
"doxygen": null,
"dracula": null,
"dsl": null,
"dtd": null,
"dtml": null,
"dtrace": null,
"dylan": null,
"dylanintr": null,
"dylanlid": null,
"ecd": null,
"edif": null,
"eiffel": null,
"elf": null,
"elinks": null,
"elmfilt": null,
"erlang": null,
"eruby": null,
"esmtprc": null,
"esqlc": null,
"esterel": null,
"eterm": null,
"eviews": null,
"exim": null,
"expect": null,
"exports": null,
"fan": null,
"fasm": null,
"fdcc": null,
"fetchmail": null,
"fgl": null,
"flexwiki": null,
"focexec": null,
"form": null,
"forth": null,
"fortran": null,
"foxpro": null,
"framescript": null,
"freebasic": null,
"fstab": null,
"fvwm": null,
"fvwm2m4": null,
"gdb": null,
"gdmo": null,
"gedcom": null,
"git": null,
"gitcommit": null,
"gitconfig": null,
"gitrebase": null,
"gitsendemail": null,
"gkrellmrc": null,
"gnuplot": null,
"gp": null,
"gpg": null,
"grads": null,
"gretl": null,
"groff": null,
"groovy": null,
"group": null,
"grub": null,
"gsp": null,
"gtkrc": null,
"haml": "Haml",
"hamster": null,
"haskell": "Haskell",
"haste": null,
"hastepreproc": null,
"hb": null,
"help": null,
"hercules": null,
"hex": null,
"hog": null,
"hostconf": null,
"html": "HTML",
"htmlcheetah": "HTML",
"htmldjango": "HTML",
"htmlm4": "HTML",
"htmlos": null,
"ia64": null,
"ibasic": null,
"icemenu": null,
"icon": null,
"idl": null,
"idlang": null,
"indent": null,
"inform": null,
"initex": null,
"initng": null,
"inittab": null,
"ipfilter": null,
"ishd": null,
"iss": null,
"ist": null,
"jal": null,
"jam": null,
"jargon": null,
"java": "Java",
"javacc": null,
"javascript": "JavaScript",
"jess": null,
"jgraph": null,
"jproperties": null,
"jsp": null,
"kconfig": null,
"kix": null,
"kscript": null,
"kwt": null,
"lace": null,
"latte": null,
"ld": null,
"ldapconf": null,
"ldif": null,
"lex": null,
"lftp": null,
"lhaskell": "Haskell",
"libao": null,
"lifelines": null,
"lilo": null,
"limits": null,
"liquid": null,
"lisp": null,
"lite": null,
"litestep": null,
"loginaccess": null,
"logindefs": null,
"logtalk": null,
"lotos": null,
"lout": null,
"lpc": null,
"lprolog": null,
"lscript": null,
"lsl": null,
"lss": null,
"lua": null,
"lynx": null,
"m4": null,
"mail": null,
"mailaliases": null,
"mailcap": null,
"make": null,
"man": null,
"manconf": null,
"manual": null,
"maple": null,
"markdown": "Markdown",
"masm": null,
"mason": null,
"master": null,
"matlab": null,
"maxima": null,
"mel": null,
"messages": null,
"mf": null,
"mgl": null,
"mgp": null,
"mib": null,
"mma": null,
"mmix": null,
"mmp": null,
"modconf": null,
"model": null,
"modsim3": null,
"modula2": null,
"modula3": null,
"monk": null,
"moo": null,
"mp": null,
"mplayerconf": null,
"mrxvtrc": null,
"msidl": null,
"msmessages": null,
"msql": null,
"mupad": null,
"mush": null,
"muttrc": null,
"mysql": null,
"named": null,
"nanorc": null,
"nasm": null,
"nastran": null,
"natural": null,
"ncf": null,
"netrc": null,
"netrw": null,
"nosyntax": null,
"nqc": null,
"nroff": null,
"nsis": null,
"obj": null,
"objc": "Objective-C",
"objcpp": "Objective-C++",
"ocaml": "OCaml",
"occam": null,
"omnimark": null,
"openroad": null,
"opl": null,
"ora": null,
"pamconf": null,
"papp": null,
"pascal": null,
"passwd": null,
"pcap": null,
"pccts": null,
"pdf": null,
"perl": "Perl",
"perl6": "Perl",
"pf": null,
"pfmain": null,
"php": "PHP",
"phtml": "PHP",
"pic": null,
"pike": null,
"pilrc": null,
"pine": null,
"pinfo": null,
"plaintex": null,
"plm": null,
"plp": null,
"plsql": null,
"po": null,
"pod": null,
"postscr": null,
"pov": null,
"povini": null,
"ppd": null,
"ppwiz": null,
"prescribe": null,
"privoxy": null,
"procmail": null,
"progress": null,
"prolog": "Prolog",
"promela": null,
"protocols": null,
"psf": null,
"ptcap": null,
"purifylog": null,
"pyrex": null,
"python": "Python",
"qf": null,
"quake": null,
"r": "R",
"racc": null,
"radiance": null,
"ratpoison": null,
"rc": null,
"rcs": null,
"rcslog": null,
"readline": null,
"rebol": null,
"registry": null,
"remind": null,
"resolv": null,
"reva": null,
"rexx": null,
"rhelp": null,
"rib": null,
"rnc": null,
"rnoweb": null,
"robots": null,
"rpcgen": null,
"rpl": null,
"rst": null,
"rtf": null,
"ruby": "Ruby",
"samba": null,
"sas": null,
"sass": "Sass",
"sather": null,
"scheme": "Scheme",
"scilab": null,
"screen": null,
"scss": "SCSS",
"sd": null,
"sdc": null,
"sdl": null,
"sed": null,
"sendpr": null,
"sensors": null,
"services": null,
"setserial": null,
"sgml": null,
"sgmldecl": null,
"sgmllnx": null,
"sh": null,
"sicad": null,
"sieve": null,
"simula": null,
"sinda": null,
"sindacmp": null,
"sindaout": null,
"sisu": null,
"skill": "SKILL",
"sl": null,
"slang": null,
"slice": null,
"slpconf": null,
"slpreg": null,
"slpspi": null,
"slrnrc": null,
"slrnsc": null,
"sm": null,
"smarty": null,
"smcl": null,
"smil": null,
"smith": null,
"sml": null,
"snnsnet": null,
"snnspat": null,
"snnsres": null,
"snobol4": null,
"spec": null,
"specman": null,
"spice": null,
"splint": null,
"spup": null,
"spyce": null,
"sql": null,
"sqlanywhere": null,
"sqlforms": null,
"sqlinformix": null,
"sqlj": null,
"sqloracle": null,
"sqr": null,
"squid": null,
"sshconfig": null,
"sshdconfig": null,
"st": null,
"stata": null,
"stp": null,
"strace": null,
"sudoers": null,
"svg": null,
"svn": null,
"syncolor": null,
"synload": null,
"syntax": null,
"sysctl": null,
"tads": null,
"tags": null,
"tak": null,
"takcmp": null,
"takout": null,
"tar": null,
"taskdata": null,
"taskedit": null,
"tasm": null,
"tcl": null,
"tcsh": null,
"terminfo": null,
"tex": null,
"texinfo": null,
"texmf": null,
"tf": null,
"tidy": null,
"tilde": null,
"tli": null,
"tpp": null,
"trasys": null,
"trustees": null,
"tsalt": null,
"tsscl": null,
"tssgm": null,
"tssop": null,
"uc": null,
"udevconf": null,
"udevperm": null,
"udevrules": null,
"uil": null,
"updatedb": null,
"valgrind": null,
"vb": "VB.net",
"vera": null,
"verilog": null,
"verilogams": null,
"vgrindefs": null,
"vhdl": null,
"vim": "VimL",
"viminfo": null,
"virata": null,
"vmasm": null,
"voscm": null,
"vrml": null,
"vsejcl": null,
"wdiff": null,
"web": null,
"webmacro": null,
"wget": null,
"winbatch": null,
"wml": null,
"wsh": null,
"wsml": null,
"wvdial": null,
"xbl": null,
"xdefaults": null,
"xf86conf": null,
"xhtml": "HTML",
"xinetd": null,
"xkb": null,
"xmath": null,
"xml": "XML",
"xmodmap": null,
"xpm": null,
"xpm2": null,
"xquery": null,
"xs": null,
"xsd": null,
"xslt": null,
"xxd": null,
"yacc": null,
"yaml": "YAML",
"z8a": null,
"zsh": null
}

View File

@ -1,115 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime.logger
~~~~~~~~~~~~~~~
Provides the configured logger for writing JSON to the log file.
:copyright: (c) 2013 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
import logging
import os
import traceback
from .compat import u
from .packages.requests.packages import urllib3
try:
from collections import OrderedDict # pragma: nocover
except ImportError: # pragma: nocover
from .packages.ordereddict import OrderedDict
try:
from .packages import simplejson as json # pragma: nocover
except (ImportError, SyntaxError): # pragma: nocover
import json
class JsonFormatter(logging.Formatter):
def setup(self, timestamp, is_write, entity, version, plugin, verbose,
warnings=False):
self.timestamp = timestamp
self.is_write = is_write
self.entity = entity
self.version = version
self.plugin = plugin
self.verbose = verbose
self.warnings = warnings
def format(self, record, *args):
data = OrderedDict([
('now', self.formatTime(record, self.datefmt)),
])
data['version'] = u(self.version)
if self.plugin:
data['plugin'] = u(self.plugin)
data['time'] = self.timestamp
if self.verbose:
data['caller'] = u(record.pathname)
data['lineno'] = record.lineno
if self.is_write:
data['is_write'] = self.is_write
data['file'] = u(self.entity)
data['level'] = record.levelname
data['message'] = u(record.getMessage() if self.warnings else record.msg)
return json.dumps(data)
def traceback(self, lvl=None):
logger = logging.getLogger('WakaTime')
if not lvl:
lvl = logger.getEffectiveLevel()
logger.log(lvl, traceback.format_exc())
def set_log_level(logger, args):
level = logging.WARN
if args.verbose:
level = logging.DEBUG
logger.setLevel(level)
def setup_logging(args, version):
urllib3.disable_warnings()
logger = logging.getLogger('WakaTime')
for handler in logger.handlers:
logger.removeHandler(handler)
set_log_level(logger, args)
logfile = args.logfile
if not logfile:
logfile = '~/.wakatime.log'
handler = logging.FileHandler(os.path.expanduser(logfile))
formatter = JsonFormatter(datefmt='%Y/%m/%d %H:%M:%S %z')
formatter.setup(
timestamp=args.timestamp,
is_write=args.is_write,
entity=args.entity,
version=version,
plugin=args.plugin,
verbose=args.verbose,
)
handler.setFormatter(formatter)
logger.addHandler(handler)
# add custom traceback logging method
logger.traceback = formatter.traceback
warnings_formatter = JsonFormatter(datefmt='%Y/%m/%d %H:%M:%S %z')
warnings_formatter.setup(
timestamp=args.timestamp,
is_write=args.is_write,
entity=args.entity,
version=version,
plugin=args.plugin,
verbose=args.verbose,
warnings=True,
)
warnings_handler = logging.FileHandler(os.path.expanduser(logfile))
warnings_handler.setFormatter(warnings_formatter)
logging.getLogger('py.warnings').addHandler(warnings_handler)
try:
logging.captureWarnings(True)
except AttributeError: # pragma: nocover
pass # Python >= 2.7 is needed to capture warnings
return logger

View File

@ -1,320 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime.main
~~~~~~~~~~~~~
Module entry point.
:copyright: (c) 2013 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
from __future__ import print_function
import base64
import logging
import os
import re
import sys
import traceback
import socket
pwd = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.dirname(pwd))
sys.path.insert(0, os.path.join(pwd, 'packages'))
from .__about__ import __version__
from .arguments import parseArguments
from .compat import u, is_py3
from .constants import (
API_ERROR,
AUTH_ERROR,
SUCCESS,
UNKNOWN_ERROR,
MALFORMED_HEARTBEAT_ERROR,
)
from .logger import setup_logging
from .offlinequeue import Queue
from .packages import requests
from .packages.requests.exceptions import RequestException
from .project import get_project_info
from .session_cache import SessionCache
from .stats import get_file_stats
from .utils import get_user_agent, should_exclude, format_file_path
try:
from .packages import simplejson as json # pragma: nocover
except (ImportError, SyntaxError): # pragma: nocover
import json
from .packages import tzlocal
log = logging.getLogger('WakaTime')
def send_heartbeat(project=None, branch=None, hostname=None, stats={}, key=None,
entity=None, timestamp=None, is_write=None, plugin=None,
offline=None, entity_type='file', hidefilenames=None,
proxy=None, api_url=None, timeout=None, **kwargs):
"""Sends heartbeat as POST request to WakaTime api server.
Returns `SUCCESS` when heartbeat was sent, otherwise returns an
error code constant.
"""
if not api_url:
api_url = 'https://api.wakatime.com/api/v1/heartbeats'
if not timeout:
timeout = 60
log.debug('Sending heartbeat to api at %s' % api_url)
data = {
'time': timestamp,
'entity': entity,
'type': entity_type,
}
if hidefilenames and entity is not None and entity_type == 'file':
for pattern in hidefilenames:
try:
compiled = re.compile(pattern, re.IGNORECASE)
if compiled.search(entity):
extension = u(os.path.splitext(data['entity'])[1])
data['entity'] = u('HIDDEN{0}').format(extension)
break
except re.error as ex:
log.warning(u('Regex error ({msg}) for include pattern: {pattern}').format(
msg=u(ex),
pattern=u(pattern),
))
if stats.get('lines'):
data['lines'] = stats['lines']
if stats.get('language'):
data['language'] = stats['language']
if stats.get('dependencies'):
data['dependencies'] = stats['dependencies']
if stats.get('lineno'):
data['lineno'] = stats['lineno']
if stats.get('cursorpos'):
data['cursorpos'] = stats['cursorpos']
if is_write:
data['is_write'] = is_write
if project:
data['project'] = project
if branch:
data['branch'] = branch
log.debug(data)
# setup api request
request_body = json.dumps(data)
api_key = u(base64.b64encode(str.encode(key) if is_py3 else key))
auth = u('Basic {api_key}').format(api_key=api_key)
headers = {
'User-Agent': get_user_agent(plugin),
'Content-Type': 'application/json',
'Accept': 'application/json',
'Authorization': auth,
}
if hostname:
headers['X-Machine-Name'] = u(hostname).encode('utf-8')
# add Olson timezone to request
try:
tz = tzlocal.get_localzone()
except:
tz = None
if tz:
headers['TimeZone'] = u(tz.zone).encode('utf-8')
session_cache = SessionCache()
session = session_cache.get()
proxies = {}
if proxy:
if '\\' in proxy:
from .packages.requests_ntlm import HttpNtlmAuth
username = proxy.rsplit(':', 1)
password = ''
if len(username) == 2:
password = username[1]
username = username[0]
session.auth = HttpNtlmAuth(username, password, session)
else:
proxies['https'] = proxy
# log time to api
response = None
try:
response = session.post(api_url, data=request_body, headers=headers,
proxies=proxies, timeout=timeout)
except RequestException:
exception_data = {
sys.exc_info()[0].__name__: u(sys.exc_info()[1]),
}
if log.isEnabledFor(logging.DEBUG):
exception_data['traceback'] = traceback.format_exc()
if offline:
queue = Queue()
queue.push(data, json.dumps(stats), plugin)
if log.isEnabledFor(logging.DEBUG):
log.warn(exception_data)
else:
log.error(exception_data)
except: # delete cached session when requests raises unknown exception
exception_data = {
sys.exc_info()[0].__name__: u(sys.exc_info()[1]),
'traceback': traceback.format_exc(),
}
if offline:
queue = Queue()
queue.push(data, json.dumps(stats), plugin)
log.warn(exception_data)
session_cache.delete()
else:
code = response.status_code if response is not None else None
content = response.text if response is not None else None
if code == requests.codes.created or code == requests.codes.accepted:
log.debug({
'response_code': code,
})
session_cache.save(session)
return SUCCESS
if offline:
if code != 400:
queue = Queue()
queue.push(data, json.dumps(stats), plugin)
if code == 401:
log.error({
'response_code': code,
'response_content': content,
})
session_cache.delete()
return AUTH_ERROR
elif log.isEnabledFor(logging.DEBUG):
log.warn({
'response_code': code,
'response_content': content,
})
else:
log.error({
'response_code': code,
'response_content': content,
})
else:
log.error({
'response_code': code,
'response_content': content,
})
session_cache.delete()
return API_ERROR
def sync_offline_heartbeats(args, hostname):
"""Sends all heartbeats which were cached in the offline Queue."""
queue = Queue()
while True:
heartbeat = queue.pop()
if heartbeat is None:
break
status = send_heartbeat(
project=heartbeat['project'],
entity=heartbeat['entity'],
timestamp=heartbeat['time'],
branch=heartbeat['branch'],
hostname=hostname,
stats=json.loads(heartbeat['stats']),
key=args.key,
is_write=heartbeat['is_write'],
plugin=heartbeat['plugin'],
offline=args.offline,
hidefilenames=args.hidefilenames,
entity_type=heartbeat['type'],
proxy=args.proxy,
api_url=args.api_url,
timeout=args.timeout,
)
if status != SUCCESS:
if status == AUTH_ERROR:
return AUTH_ERROR
break
return SUCCESS
def process_heartbeat(args, configs, hostname, heartbeat):
exclude = should_exclude(heartbeat['entity'], args.include, args.exclude)
if exclude is not False:
log.debug(u('Skipping because matches exclude pattern: {pattern}').format(
pattern=u(exclude),
))
return SUCCESS
if heartbeat.get('entity_type') not in ['file', 'domain', 'app']:
heartbeat['entity_type'] = 'file'
if heartbeat['entity_type'] == 'file':
heartbeat['entity'] = format_file_path(heartbeat['entity'])
if heartbeat['entity_type'] != 'file' or os.path.isfile(heartbeat['entity']):
stats = get_file_stats(heartbeat['entity'],
entity_type=heartbeat['entity_type'],
lineno=heartbeat.get('lineno'),
cursorpos=heartbeat.get('cursorpos'),
plugin=args.plugin,
language=heartbeat.get('language'))
project = heartbeat.get('project') or heartbeat.get('alternate_project')
branch = None
if heartbeat['entity_type'] == 'file':
project, branch = get_project_info(configs, heartbeat)
heartbeat['project'] = project
heartbeat['branch'] = branch
heartbeat['stats'] = stats
heartbeat['hostname'] = hostname
heartbeat['timeout'] = args.timeout
heartbeat['key'] = args.key
heartbeat['plugin'] = args.plugin
heartbeat['offline'] = args.offline
heartbeat['hidefilenames'] = args.hidefilenames
heartbeat['proxy'] = args.proxy
heartbeat['api_url'] = args.api_url
return send_heartbeat(**heartbeat)
else:
log.debug('File does not exist; ignoring this heartbeat.')
return SUCCESS
def execute(argv=None):
if argv:
sys.argv = ['wakatime'] + argv
args, configs = parseArguments()
setup_logging(args, __version__)
try:
hostname = args.hostname or socket.gethostname()
heartbeat = vars(args)
retval = process_heartbeat(args, configs, hostname, heartbeat)
if args.extra_heartbeats:
try:
for heartbeat in json.loads(sys.stdin.readline()):
retval = process_heartbeat(args, configs, hostname, heartbeat)
except json.JSONDecodeError:
retval = MALFORMED_HEARTBEAT_ERROR
if retval == SUCCESS:
retval = sync_offline_heartbeats(args, hostname)
return retval
except:
log.traceback(logging.ERROR)
print(traceback.format_exc())
return UNKNOWN_ERROR

View File

@ -1,128 +0,0 @@
# -*- coding: utf-8 -*-
"""
wakatime.offlinequeue
~~~~~~~~~~~~~~~~~~~~~
Queue for saving heartbeats while offline.
:copyright: (c) 2014 Alan Hamlett.
:license: BSD, see LICENSE for more details.
"""
import logging
import os
from time import sleep
try:
import sqlite3
HAS_SQL = True
except ImportError: # pragma: nocover
HAS_SQL = False
from .compat import u
log = logging.getLogger('WakaTime')
class Queue(object):
db_file = os.path.join(os.path.expanduser('~'), '.wakatime.db')
table_name = 'heartbeat_1'
def get_db_file(self):
return self.db_file
def connect(self):
conn = sqlite3.connect(self.get_db_file(), isolation_level=None)
c = conn.cursor()
c.execute('''CREATE TABLE IF NOT EXISTS {0} (
entity text,
type text,
time real,
project text,
branch text,
is_write integer,
stats text,
misc text,
plugin text)
'''.format(self.table_name))
return (conn, c)
def push(self, data, stats, plugin, misc=None):
if not HAS_SQL: # pragma: nocover
return
try:
conn, c = self.connect()
heartbeat = {
'entity': u(data.get('entity')),
'type': u(data.get('type')),
'time': data.get('time'),
'project': u(data.get('project')),
'branch': u(data.get('branch')),
'is_write': 1 if data.get('is_write') else 0,
'stats': u(stats),
'misc': u(misc),
'plugin': u(plugin),
}
c.execute('INSERT INTO {0} VALUES (:entity,:type,:time,:project,:branch,:is_write,:stats,:misc,:plugin)'.format(self.table_name), heartbeat)
conn.commit()
conn.close()
except sqlite3.Error:
log.traceback()
def pop(self):
if not HAS_SQL: # pragma: nocover
return None
tries = 3
wait = 0.1
heartbeat = None
try:
conn, c = self.connect()
except sqlite3.Error:
log.traceback(logging.DEBUG)
return None
loop = True
while loop and tries > -1:
try:
c.execute('BEGIN IMMEDIATE')
c.execute('SELECT * FROM {0} LIMIT 1'.format(self.table_name))
row = c.fetchone()
if row is not None:
values = []
clauses = []
index = 0
for row_name in ['entity', 'type', 'time', 'project', 'branch', 'is_write']:
if row[index] is not None:
clauses.append('{0}=?'.format(row_name))
values.append(row[index])
else: # pragma: nocover
clauses.append('{0} IS NULL'.format(row_name))
index += 1
if len(values) > 0:
c.execute('DELETE FROM {0} WHERE {1}'.format(self.table_name, ' AND '.join(clauses)), values)
else: # pragma: nocover
c.execute('DELETE FROM {0} WHERE {1}'.format(self.table_name, ' AND '.join(clauses)))
conn.commit()
if row is not None:
heartbeat = {
'entity': row[0],
'type': row[1],
'time': row[2],
'project': row[3],
'branch': row[4],
'is_write': True if row[5] is 1 else False,
'stats': row[6],
'misc': row[7],
'plugin': row[8],
}
loop = False
except sqlite3.Error: # pragma: nocover
log.traceback(logging.DEBUG)
sleep(wait)
tries -= 1
try:
conn.close()
except sqlite3.Error: # pragma: nocover
log.traceback(logging.DEBUG)
return heartbeat

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1,171 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from collections import MutableMapping
try:
from collections import UserDict
except ImportError:
from UserDict import UserDict
try:
from collections import OrderedDict
except ImportError:
from ordereddict import OrderedDict
from io import open
import sys
try:
from thread import get_ident
except ImportError:
try:
from _thread import get_ident
except ImportError:
from _dummy_thread import get_ident
PY2 = sys.version_info[0] == 2
PY3 = sys.version_info[0] == 3
str = type('str')
def from_none(exc):
"""raise from_none(ValueError('a')) == raise ValueError('a') from None"""
exc.__cause__ = None
exc.__suppress_context__ = True
return exc
# from reprlib 3.2.1
def recursive_repr(fillvalue='...'):
'Decorator to make a repr function return fillvalue for a recursive call'
def decorating_function(user_function):
repr_running = set()
def wrapper(self):
key = id(self), get_ident()
if key in repr_running:
return fillvalue
repr_running.add(key)
try:
result = user_function(self)
finally:
repr_running.discard(key)
return result
# Can't use functools.wraps() here because of bootstrap issues
wrapper.__module__ = getattr(user_function, '__module__')
wrapper.__doc__ = getattr(user_function, '__doc__')
wrapper.__name__ = getattr(user_function, '__name__')
wrapper.__annotations__ = getattr(user_function, '__annotations__', {})
return wrapper
return decorating_function
# from collections 3.2.1
class _ChainMap(MutableMapping):
''' A ChainMap groups multiple dicts (or other mappings) together
to create a single, updateable view.
The underlying mappings are stored in a list. That list is public and can
accessed or updated using the *maps* attribute. There is no other state.
Lookups search the underlying mappings successively until a key is found.
In contrast, writes, updates, and deletions only operate on the first
mapping.
'''
def __init__(self, *maps):
'''Initialize a ChainMap by setting *maps* to the given mappings.
If no mappings are provided, a single empty dictionary is used.
'''
self.maps = list(maps) or [{}] # always at least one map
def __missing__(self, key):
raise KeyError(key)
def __getitem__(self, key):
for mapping in self.maps:
try:
return mapping[key] # can't use 'key in mapping' with defaultdict
except KeyError:
pass
return self.__missing__(key) # support subclasses that define __missing__
def get(self, key, default=None):
return self[key] if key in self else default
def __len__(self):
return len(set().union(*self.maps)) # reuses stored hash values if possible
def __iter__(self):
return iter(set().union(*self.maps))
def __contains__(self, key):
return any(key in m for m in self.maps)
@recursive_repr()
def __repr__(self):
return '{0.__class__.__name__}({1})'.format(
self, ', '.join(map(repr, self.maps)))
@classmethod
def fromkeys(cls, iterable, *args):
'Create a ChainMap with a single dict created from the iterable.'
return cls(dict.fromkeys(iterable, *args))
def copy(self):
'New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]'
return self.__class__(self.maps[0].copy(), *self.maps[1:])
__copy__ = copy
def new_child(self): # like Django's Context.push()
'New ChainMap with a new dict followed by all previous maps.'
return self.__class__({}, *self.maps)
@property
def parents(self): # like Django's Context.pop()
'New ChainMap from maps[1:].'
return self.__class__(*self.maps[1:])
def __setitem__(self, key, value):
self.maps[0][key] = value
def __delitem__(self, key):
try:
del self.maps[0][key]
except KeyError:
raise KeyError('Key not found in the first mapping: {!r}'.format(key))
def popitem(self):
'Remove and return an item pair from maps[0]. Raise KeyError is maps[0] is empty.'
try:
return self.maps[0].popitem()
except KeyError:
raise KeyError('No keys found in the first mapping.')
def pop(self, key, *args):
'Remove *key* from maps[0] and return its value. Raise KeyError if *key* not in maps[0].'
try:
return self.maps[0].pop(key, *args)
except KeyError:
raise KeyError('Key not found in the first mapping: {!r}'.format(key))
def clear(self):
'Clear maps[0], leaving maps[1:] intact.'
self.maps[0].clear()
try:
from collections import ChainMap
except ImportError:
ChainMap = _ChainMap

View File

@ -1,156 +0,0 @@
# This file is part of 'NTLM Authorization Proxy Server' http://sourceforge.net/projects/ntlmaps/
# Copyright 2001 Dmitry A. Rozmanov <dima@xenon.spb.ru>
#
# This library is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
from __future__ import division
import six
C = 0x1000000000
def norm(n):
return n & 0xFFFFFFFF
class U32:
v = 0
def __init__(self, value=0):
if not isinstance(value, six.integer_types):
value = six.byte2int(value)
self.v = C + norm(abs(int(value)))
def set(self, value=0):
self.v = C + norm(abs(int(value)))
def __repr__(self):
return hex(norm(self.v))
def __long__(self):
return int(norm(self.v))
def __int__(self):
return int(norm(self.v))
def __chr__(self):
return chr(norm(self.v))
def __add__(self, b):
r = U32()
r.v = C + norm(self.v + b.v)
return r
def __sub__(self, b):
r = U32()
if self.v < b.v:
r.v = C + norm(0x100000000 - (b.v - self.v))
else:
r.v = C + norm(self.v - b.v)
return r
def __mul__(self, b):
r = U32()
r.v = C + norm(self.v * b.v)
return r
def __div__(self, b):
r = U32()
r.v = C + (norm(self.v) // norm(b.v))
return r
def __truediv__(self, b):
r = U32()
r.v = C + (norm(self.v) / norm(b.v))
return r
def __mod__(self, b):
r = U32()
r.v = C + (norm(self.v) % norm(b.v))
return r
def __neg__(self):
return U32(self.v)
def __pos__(self):
return U32(self.v)
def __abs__(self):
return U32(self.v)
def __invert__(self):
r = U32()
r.v = C + norm(~self.v)
return r
def __lshift__(self, b):
r = U32()
r.v = C + norm(self.v << b)
return r
def __rshift__(self, b):
r = U32()
r.v = C + (norm(self.v) >> b)
return r
def __and__(self, b):
r = U32()
r.v = C + norm(self.v & b.v)
return r
def __or__(self, b):
r = U32()
r.v = C + norm(self.v | b.v)
return r
def __xor__(self, b):
r = U32()
r.v = C + norm(self.v ^ b.v)
return r
def __not__(self):
return U32(not norm(self.v))
def truth(self):
return norm(self.v)
def __cmp__(self, b):
if norm(self.v) > norm(b.v):
return 1
elif norm(self.v) < norm(b.v):
return -1
else:
return 0
def __lt__(self, other):
return self.v < other.v
def __gt__(self, other):
return self.v > other.v
def __eq__(self, other):
return self.v == other.v
def __le__(self, other):
return self.v <= other.v
def __ge__(self, other):
return self.v >= other.v
def __ne__(self, other):
return self.v != other.v
def __nonzero__(self):
return norm(self.v)

View File

@ -1,3 +0,0 @@
from . import ntlm, session_security
__all__ = ('ntlm', 'session_security')

View File

@ -1,80 +0,0 @@
# This library is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
import binascii
import hashlib
import hmac
import re
from ntlm_auth import des
def _lmowfv1(password):
"""
[MS-NLMP] v28.0 2016-07-14
3.3.1 NTLM v1 Authentication
Same function as LMOWFv1 in document to create a one way hash of the password. Only
used in NTLMv1 auth without session security
:param password: The password of the user we are trying to authenticate with
:return res: A Lan Manager hash of the password supplied
"""
# fix the password length to 14 bytes
password = password.upper()
lm_pw = password[0:14]
# do hash
magic_str = b"KGS!@#$%" # page 56 in [MS-NLMP v28.0]
res = b''
dobj = des.DES(lm_pw[0:7])
res = res + dobj.encrypt(magic_str)
dobj = des.DES(lm_pw[7:14])
res = res + dobj.encrypt(magic_str)
return res
def _ntowfv1(password):
"""
[MS-NLMP] v28.0 2016-07-14
3.3.1 NTLM v1 Authentication
Same function as NTOWFv1 in document to create a one way hash of the password. Only
used in NTLMv1 auth without session security
:param password: The password of the user we are trying to authenticate with
:return digest: An NT hash of the password supplied
"""
digest = hashlib.new('md4', password.encode('utf-16le')).digest()
return digest
def _ntowfv2(user_name, password, domain_name):
"""
[MS-NLMP] v28.0 2016-07-14
3.3.2 NTLM v2 Authentication
Same function as NTOWFv2 (and LMOWFv2) in document to create a one way hash of the password.
This combines some extra security features over the v1 calculations used in NTLMv2 auth.
:param user_name: The user name of the user we are trying to authenticate with
:param password: The password of the user we are trying to authenticate with
:param domain_name: The domain name of the user account we are authenticated with
:return digest: An NT hash of the parameters supplied
"""
digest = _ntowfv1(password)
digest = hmac.new(digest, (user_name.upper() + domain_name).encode('utf-16le')).digest()
return digest

View File

@ -1,138 +0,0 @@
# This library is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
import binascii
import hashlib
import hmac
from ntlm_auth import des
from ntlm_auth.constants import NegotiateFlags
def _get_exchange_key_ntlm_v1(negotiate_flags, session_base_key, server_challenge, lm_challenge_response, lm_hash):
"""
[MS-NLMP] v28.0 2016-07-14
4.3.5.1 KXKEY
Calculates the Key Exchange Key for NTLMv1 authentication. Used for signing and sealing messages
@param negotiate_flags:
@param session_base_key: A session key calculated from the user password challenge
@param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
@param lm_challenge_response: The LmChallengeResponse value computed in ComputeResponse
@param lm_hash: The LMOWF computed in Compute Response
@return key_exchange_key: The Key Exchange Key (KXKEY) used to sign and seal messages and compute the ExportedSessionKey
"""
if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY:
key_exchange_key = hmac.new(session_base_key, server_challenge + lm_challenge_response[:8]).digest()
elif negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_LM_KEY:
des_handler = des.DES(lm_hash[:7])
first_des = des_handler.encrypt(lm_challenge_response[:8])
des_handler = des.DES(lm_hash[7:8] + binascii.unhexlify('bdbdbdbdbdbdbd'))
second_des = des_handler.encrypt(lm_challenge_response[:8])
key_exchange_key = first_des + second_des
elif negotiate_flags & NegotiateFlags.NTLMSSP_REQUEST_NON_NT_SESSION_KEY:
key_exchange_key = lm_hash[:8] + b'\0' * 8
else:
key_exchange_key = session_base_key
return key_exchange_key
def _get_exchange_key_ntlm_v2(session_base_key):
"""
[MS-NLMP] v28.0 2016-07-14
4.3.5.1 KXKEY
Calculates the Key Exchange Key for NTLMv2 authentication. Used for signing and sealing messages.
According to docs, 'If NTLM v2 is used, KeyExchangeKey MUST be set to the given 128-bit SessionBaseKey
@param session_base_key: A session key calculated from the user password challenge
@return key_exchange_key: The Key Exchange Key (KXKEY) used to sign and seal messages
"""
return session_base_key
def get_sign_key(exported_session_key, magic_constant):
"""
3.4.5.2 SIGNKEY
@param exported_session_key: A 128-bit session key used to derive signing and sealing keys
@param magic_constant: A constant value set in the MS-NLMP documentation (constants.SignSealConstants)
@return sign_key: Key used to sign messages
"""
sign_key = hashlib.md5(exported_session_key + magic_constant).digest()
return sign_key
def get_seal_key(negotiate_flags, exported_session_key, magic_constant):
"""
3.4.5.3. SEALKEY
Main method to use to calculate the seal_key used to seal (encrypt) messages. This will determine
the correct method below to use based on the compatibility flags set and should be called instead
of the others
@param exported_session_key: A 128-bit session key used to derive signing and sealing keys
@param negotiate_flags: The negotiate_flags structure sent by the server
@param magic_constant: A constant value set in the MS-NLMP documentation (constants.SignSealConstants)
@return seal_key: Key used to seal messages
"""
if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY:
seal_key = _get_seal_key_ntlm2(negotiate_flags, exported_session_key, magic_constant)
elif negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_LM_KEY:
seal_key = _get_seal_key_ntlm1(negotiate_flags, exported_session_key)
else:
seal_key = exported_session_key
return seal_key
def _get_seal_key_ntlm1(negotiate_flags, exported_session_key):
"""
3.4.5.3 SEALKEY
Calculates the seal_key used to seal (encrypt) messages. This for authentication where
NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY has not been negotiated. Will weaken the keys
if NTLMSSP_NEGOTIATE_56 is not negotiated it will default to the 40-bit key
@param negotiate_flags: The negotiate_flags structure sent by the server
@param exported_session_key: A 128-bit session key used to derive signing and sealing keys
@return seal_key: Key used to seal messages
"""
if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_56:
seal_key = exported_session_key[:7] + binascii.unhexlify('a0')
else:
seal_key = exported_session_key[:5] + binascii.unhexlify('e538b0')
return seal_key
def _get_seal_key_ntlm2(negotiate_flags, exported_session_key, magic_constant):
"""
3.4.5.3 SEALKEY
Calculates the seal_key used to seal (encrypt) messages. This for authentication where
NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY has been negotiated. Will weaken the keys
if NTLMSSP_NEGOTIATE_128 is not negotiated, will try NEGOTIATE_56 and then will default
to the 40-bit key
@param negotiate_flags: The negotiate_flags structure sent by the server
@param exported_session_key: A 128-bit session key used to derive signing and sealing keys
@param magic_constant: A constant value set in the MS-NLMP documentation (constants.SignSealConstants)
@return seal_key: Key used to seal messages
"""
if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_128:
seal_key = exported_session_key
elif negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_56:
seal_key = exported_session_key[:7]
else:
seal_key = exported_session_key[:5]
seal_key = hashlib.md5(seal_key + magic_constant).digest()
return seal_key

View File

@ -1,397 +0,0 @@
# This library is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
import base64
import calendar
import hashlib
import hmac
import os
import struct
import time
import ntlm_auth.compute_hash as comphash
import ntlm_auth.compute_keys as compkeys
from ntlm_auth import des
from ntlm_auth.constants import NegotiateFlags, AvFlags
from ntlm_auth.gss_channel_bindings import GssChannelBindingsStruct
from ntlm_auth.target_info import TargetInfo
class ComputeResponse():
"""
Constructor for the response computations. This class will compute the various
nt and lm challenge responses.
:param user_name: The user name of the user we are trying to authenticate with
:param password: The password of the user we are trying to authenticate with
:param domain_name: The domain name of the user account we are authenticated with, default is None
:param challenge_message: A ChallengeMessage object that was received from the server after the negotiate_message
:param ntlm_compatibility: The Lan Manager Compatibility Level, used to determine what NTLM auth version to use, see Ntlm in ntlm.py for more details
"""
def __init__(self, user_name, password, domain_name, challenge_message, ntlm_compatibility):
self._user_name = user_name
self._password = password
self._domain_name = domain_name
self._challenge_message = challenge_message
self._negotiate_flags = challenge_message.negotiate_flags
self._server_challenge = challenge_message.server_challenge
self._server_target_info = challenge_message.target_info
self._ntlm_compatibility = ntlm_compatibility
self._client_challenge = os.urandom(8)
def get_lm_challenge_response(self):
"""
[MS-NLMP] v28.0 2016-07-14
3.3.1 - NTLM v1 Authentication
3.3.2 - NTLM v2 Authentication
This method returns the LmChallengeResponse key based on the ntlm_compatibility chosen
and the target_info supplied by the CHALLENGE_MESSAGE. It is quite different from what
is set in the document as it combines the NTLMv1, NTLM2 and NTLMv2 methods into one
and calls separate methods based on the ntlm_compatibility flag chosen.
:return: response (LmChallengeResponse) - The LM response to the server challenge. Computed by the client
"""
if self._negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY and self._ntlm_compatibility < 3:
response = ComputeResponse._get_LMv1_with_session_security_response(self._client_challenge)
elif 0 <= self._ntlm_compatibility <= 1:
response = ComputeResponse._get_LMv1_response(self._password, self._server_challenge)
elif self._ntlm_compatibility == 2:
# Based on the compatibility level we don't want to use LM responses, ignore the session_base_key as it is returned in nt
response, ignore_key = ComputeResponse._get_NTLMv1_response(self._password, self._server_challenge)
else:
"""
[MS-NLMP] v28.0 page 45 - 2016-07-14
3.1.5.12 Client Received a CHALLENGE_MESSAGE from the Server
If NTLMv2 authentication is used and the CHALLENGE_MESSAGE TargetInfo field has an MsvAvTimestamp present,
the client SHOULD NOT send the LmChallengeResponse and SHOULD send Z(24) instead.
"""
response = ComputeResponse._get_LMv2_response(self._user_name, self._password, self._domain_name,
self._server_challenge,
self._client_challenge)
if self._server_target_info is not None:
timestamp = self._server_target_info[TargetInfo.MSV_AV_TIMESTAMP]
if timestamp is not None:
response = b'\0' * 24
return response
def get_nt_challenge_response(self, lm_challenge_response, server_certificate_hash):
"""
[MS-NLMP] v28.0 2016-07-14
3.3.1 - NTLM v1 Authentication
3.3.2 - NTLM v2 Authentication
This method returns the NtChallengeResponse key based on the ntlm_compatibility chosen
and the target_info supplied by the CHALLENGE_MESSAGE. It is quite different from what
is set in the document as it combines the NTLMv1, NTLM2 and NTLMv2 methods into one
and calls separate methods based on the ntlm_compatibility value chosen.
:param lm_challenge_response: The LmChallengeResponse calculated beforeand, used to get the key_exchange_key value
:param server_certificate_hash: The SHA256 hash of the server certificate (DER encoded) NTLM is authenticated to.
Used in Channel Binding Tokens if present, default value is None. See
AuthenticateMessage in messages.py for more details
:return response: (NtChallengeResponse) - The NT response to the server challenge. Computed by the client
:return session_base_key: (SessionBaseKey) - A session key calculated from the user password challenge
:return target_info: (AV_PAIR) - The AV_PAIR structure used in the nt_challenge calculations
"""
if self._negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY and self._ntlm_compatibility < 3:
# The compatibility level is less than 3 which means it doesn't support NTLMv2 but we want extended security so use NTLM2 which is different from NTLMv2
# [MS-NLMP] - 3.3.1 NTLMv1 Authentication
response, session_base_key = ComputeResponse._get_NTLM2_response(self._password, self._server_challenge, self._client_challenge)
key_exchange_key = compkeys._get_exchange_key_ntlm_v1(self._negotiate_flags, session_base_key,
self._server_challenge, lm_challenge_response,
comphash._lmowfv1(self._password))
target_info = None
elif 0 <= self._ntlm_compatibility < 3:
response, session_base_key = ComputeResponse._get_NTLMv1_response(self._password, self._server_challenge)
key_exchange_key = compkeys._get_exchange_key_ntlm_v1(self._negotiate_flags, session_base_key,
self._server_challenge, lm_challenge_response,
comphash._lmowfv1(self._password))
target_info = None
else:
if self._server_target_info is None:
target_info = TargetInfo()
else:
target_info = self._server_target_info
if target_info[TargetInfo.MSV_AV_TIMESTAMP] is None:
timestamp = get_windows_timestamp()
else:
timestamp = target_info[TargetInfo.MSV_AV_TIMESTAMP][1]
# [MS-NLMP] If the CHALLENGE_MESSAGE TargetInfo field has an MsvAvTimestamp present, the client SHOULD provide a MIC
target_info[TargetInfo.MSV_AV_FLAGS] = struct.pack("<L", AvFlags.MIC_PROVIDED)
if server_certificate_hash != None:
channel_bindings_hash = ComputeResponse._get_channel_bindings_value(server_certificate_hash)
target_info[TargetInfo.MSV_AV_CHANNEL_BINDINGS] = channel_bindings_hash
response, session_base_key = ComputeResponse._get_NTLMv2_response(self._user_name, self._password, self._domain_name,
self._server_challenge, self._client_challenge, timestamp, target_info)
key_exchange_key = compkeys._get_exchange_key_ntlm_v2(session_base_key)
return response, key_exchange_key, target_info
@staticmethod
def _get_LMv1_response(password, server_challenge):
"""
[MS-NLMP] v28.0 2016-07-14
2.2.2.3 LM_RESPONSE
The LM_RESPONSE structure defines the NTLM v1 authentication LmChallengeResponse
in the AUTHENTICATE_MESSAGE. This response is used only when NTLM v1
authentication is configured.
:param password: The password of the user we are trying to authenticate with
:param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
:return response: LmChallengeResponse to the server challenge
"""
lm_hash = comphash._lmowfv1(password)
response = ComputeResponse._calc_resp(lm_hash, server_challenge)
return response
@staticmethod
def _get_LMv1_with_session_security_response(client_challenge):
"""
[MS-NLMP] v28.0 2016-07-14
2.2.2.3 LM_RESPONSE
The LM_RESPONSE structure defines the NTLM v1 authentication LmChallengeResponse
in the AUTHENTICATE_MESSAGE. This response is used only when NTLM v1
authentication is configured and NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY is flages.
:param client_challenge: A random 8-byte response generated by the client for the AUTHENTICATE_MESSAGE
:return response: LmChallengeResponse to the server challenge
"""
response = client_challenge + b'\0' * 16
return response
@staticmethod
def _get_LMv2_response(user_name, password, domain_name, server_challenge, client_challenge):
"""
[MS-NLMP] v28.0 2016-07-14
2.2.2.4 LMv2_RESPONSE
The LMv2_RESPONSE structure defines the NTLM v2 authentication LmChallengeResponse
in the AUTHENTICATE_MESSAGE. This response is used only when NTLM v2
authentication is configured.
:param user_name: The user name of the user we are trying to authenticate with
:param password: The password of the user we are trying to authenticate with
:param domain_name: The domain name of the user account we are authenticated with
:param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
:param client_challenge: A random 8-byte response generated by the client for the AUTHENTICATE_MESSAGE
:return response: LmChallengeResponse to the server challenge
"""
nt_hash = comphash._ntowfv2(user_name, password, domain_name)
lm_hash = hmac.new(nt_hash, (server_challenge + client_challenge)).digest()
response = lm_hash + client_challenge
return response
@staticmethod
def _get_NTLMv1_response(password, server_challenge):
"""
[MS-NLMP] v28.0 2016-07-14
2.2.2.6 NTLM v1 Response: NTLM_RESPONSE
The NTLM_RESPONSE strucutre defines the NTLM v1 authentication NtChallengeResponse
in the AUTHENTICATE_MESSAGE. This response is only used when NTLM v1 authentication
is configured.
:param password: The password of the user we are trying to authenticate with
:param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
:return response: NtChallengeResponse to the server_challenge
:return session_base_key: A session key calculated from the user password challenge
"""
ntlm_hash = comphash._ntowfv1(password)
response = ComputeResponse._calc_resp(ntlm_hash, server_challenge)
session_base_key = hashlib.new('md4', ntlm_hash).digest()
return response, session_base_key
@staticmethod
def _get_NTLM2_response(password, server_challenge, client_challenge):
"""
[MS-NLMP] v28.0 2016-07-14
This name is really misleading as it isn't NTLM v2 authentication rather
This authentication is only used when the ntlm_compatibility level is set
to a value < 3 (No NTLMv2 auth) but the NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY
flag is set in the negotiate flags section. The documentation for computing this
value is on page 56 under section 3.3.1 NTLM v1 Authentication
:param password: The password of the user we are trying to authenticate with
:param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
:param client_challenge: A random 8-byte response generated by the client for the AUTHENTICATE_MESSAGE
:return response: NtChallengeResponse to the server_challenge
:return session_base_key: A session key calculated from the user password challenge
"""
ntlm_hash = comphash._ntowfv1(password)
nt_session_hash = hashlib.md5(server_challenge + client_challenge).digest()[:8]
response = ComputeResponse._calc_resp(ntlm_hash, nt_session_hash[0:8])
session_base_key = hashlib.new('md4', ntlm_hash).digest()
return response, session_base_key
@staticmethod
def _get_NTLMv2_response(user_name, password, domain_name, server_challenge, client_challenge, timestamp, target_info):
"""
[MS-NLMP] v28.0 2016-07-14
2.2.2.8 NTLM V2 Response: NTLMv2_RESPONSE
The NTLMv2_RESPONSE strucutre defines the NTLMv2 authentication NtChallengeResponse
in the AUTHENTICATE_MESSAGE. This response is used only when NTLMv2 authentication
is configured.
The guide on how this is computed is in 3.3.2 NTLM v2 Authentication.
:param user_name: The user name of the user we are trying to authenticate with
:param password: The password of the user we are trying to authenticate with
:param domain_name: The domain name of the user account we are authenticated with
:param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
:param client_challenge: A random 8-byte response generated by the client for the AUTHENTICATE_MESSAGE
:param timestamp: An 8-byte timestamp in windows format, 100 nanoseconds since 1601-01-01
:param target_info: The target_info structure from the CHALLENGE_MESSAGE with the CBT attached if required
:return response: NtChallengeResponse to the server_challenge
:return session_base_key: A session key calculated from the user password challenge
"""
nt_hash = comphash._ntowfv2(user_name, password, domain_name)
temp = ComputeResponse._get_NTLMv2_temp(timestamp, client_challenge, target_info)
nt_proof_str = hmac.new(nt_hash, (server_challenge + temp)).digest()
response = nt_proof_str + temp
session_base_key = hmac.new(nt_hash, nt_proof_str).digest()
return response, session_base_key
@staticmethod
def _get_NTLMv2_temp(timestamp, client_challenge, target_info):
"""
[MS-NLMP] v28.0 2016-07-14
2.2.2.7 NTLMv2_CLIENT_CHALLENGE - variable length
The NTLMv2_CLIENT_CHALLENGE structure defines the client challenge in
the AUTHENTICATE_MESSAGE. This structure is used only when NTLM v2
authentication is configured and is transported in the NTLMv2_RESPONSE
structure.
The method to create this structure is defined in 3.3.2 NTLMv2 Authentication.
In this method this variable is known as the temp value. The target_info variable
corresponds to the ServerName variable used in that documentation. This is in
reality a lot more than just the ServerName and contains the AV_PAIRS structure
we need to transport with the message like Channel Binding tokens and others.
By default this will be the target_info returned from the CHALLENGE_MESSAGE plus
MSV_AV_CHANNEL_BINDINGS if specified otherwise it is a new target_info set with
MSV_AV_TIMESTAMP to the current time.
:param timestamp: An 8-byte timestamp in windows format, 100 nanoseconds since 1601-01-01
:param client_challenge: A random 8-byte response generated by the client for the AUTHENTICATE_MESSAGE
:param target_info: The target_info structure from the CHALLENGE_MESSAGE with the CBT attached if required
:return temp: The CLIENT_CHALLENGE structure that will be added to the NtChallengeResponse structure
"""
resp_type = b'\1'
hi_resp_type = b'\1'
reserved1 = b'\0' * 2
reserved2 = b'\0' * 4
reserved3 = b'\0' * 4
reserved4 = b'\0' * 4 # This byte is not in the structure defined in 2.2.2.7 but is in the computation guide, works with it present
temp = resp_type + hi_resp_type + reserved1 + \
reserved2 + \
timestamp + \
client_challenge + \
reserved3 + \
target_info.get_data() + reserved4
return temp
@staticmethod
def _calc_resp(password_hash, server_challenge):
"""
Generate the LM response given a 16-byte password hash and the challenge
from the CHALLENGE_MESSAGE
:param password_hash: A 16-byte password hash
:param server_challenge: A random 8-byte response generated by the server in the CHALLENGE_MESSAGE
:return res: A 24-byte buffer to contain the LM response upon return
"""
# padding with zeros to make the hash 21 bytes long
password_hash += b'\0' * (21 - len(password_hash))
res = b''
dobj = des.DES(password_hash[0:7])
res = res + dobj.encrypt(server_challenge[0:8])
dobj = des.DES(password_hash[7:14])
res = res + dobj.encrypt(server_challenge[0:8])
dobj = des.DES(password_hash[14:21])
res = res + dobj.encrypt(server_challenge[0:8])
return res
@staticmethod
def _get_channel_bindings_value(server_certificate_hash):
"""
https://msdn.microsoft.com/en-us/library/windows/desktop/dd919963%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
https://blogs.msdn.microsoft.com/openspecification/2013/03/26/ntlm-and-channel-binding-hash-aka-extended-protection-for-authentication/
Get's the MD5 hash of the gss_channel_bindings_struct to add to the AV_PAIR MSV_AV_CHANNEL_BINDINGS.
This method takes in the SHA256 hash (Hash of the DER encoded certificate of the server we are connecting to)
and add's it to the gss_channel_bindings_struct. It then gets the MD5 hash and converts this to a
byte array in preparation of adding it to the AV_PAIR structure.
:param server_certificate_hash: The SHA256 hash of the server certificate (DER encoded) NTLM is authenticated to
:return channel_bindings: An MD5 hash of the gss_channel_bindings_struct to add to the AV_PAIR MsvChannelBindings
"""
# Channel Binding Tokens support, used for NTLMv2
# Decode the SHA256 certificate hash
certificate_digest = base64.b16decode(server_certificate_hash)
# Initialise the GssChannelBindingsStruct and add the certificate_digest to the application_data field
gss_channel_bindings = GssChannelBindingsStruct()
gss_channel_bindings[gss_channel_bindings.APPLICATION_DATA] = 'tls-server-end-point:'.encode() + certificate_digest
# Get the gss_channel_bindings_struct and create an MD5 hash
channel_bindings_struct_data = gss_channel_bindings.get_data()
channel_bindings_hash = hashlib.md5(channel_bindings_struct_data).hexdigest()
try:
cbt_value = bytearray.fromhex(channel_bindings_hash)
except TypeError:
# Work-around for Python 2.6 bug
cbt_value = bytearray.fromhex(unicode(channel_bindings_hash))
channel_bindings = bytes(cbt_value)
return channel_bindings
def get_windows_timestamp():
# Get Windows Date time, 100 nanoseconds since 1601-01-01 in a 64 bit structure
timestamp = struct.pack('<q', (116444736000000000 + calendar.timegm(time.gmtime()) * 10000000))
return timestamp

View File

@ -1,92 +0,0 @@
# This library is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
"""
[MS-NLMP] v28.0 2016-07-14
2.2 Message Syntax
The signature field used in NTLM messages
"""
NTLM_SIGNATURE = b'NTLMSSP\0'
"""
[MS-NLMP] v28.0 2016-07-14
2.2 Message Syntax
The 3 message type options you can have in a message.
"""
class MessageTypes(object):
NTLM_NEGOTIATE = 0x1
NTLM_CHALLENGE = 0x2
NTLM_AUTHENTICATE = 0x3
"""
[MS-NLMP] v28.0 2016-07-14
2.2.2.1 AV_PAIR (MsvAvFlags)
A 32-bit value indicated server or client configuration
"""
class AvFlags(object):
AUTHENTICATION_CONSTRAINED = 0x1
MIC_PROVIDED = 0x2
UNTRUSTED_SPN_SOURCE = 0x4
"""
[MS-NLMP] v28.0 2016-07-14
2.2.2.5 NEGOTIATE
During NTLM authentication, each of the following flags is a possible value of the
NegotiateFlags field of the NEGOTIATE_MESSAGE, CHALLENGE_MESSAGE and AUTHENTICATE_MESSAGE,
unless otherwise noted. These flags define client or server NTLM capabilities
supported by the sender.
"""
class NegotiateFlags(object):
NTLMSSP_NEGOTIATE_56 = 0x80000000
NTLMSSP_NEGOTIATE_KEY_EXCH = 0x40000000
NTLMSSP_NEGOTIATE_128 = 0x20000000
NTLMSSP_RESERVED_R1 = 0x10000000
NTLMSSP_RESERVED_R2 = 0x08000000
NTLMSSP_RESERVED_R3 = 0x04000000
NTLMSSP_NEGOTIATE_VERSION = 0x02000000
NTLMSSP_RESERVED_R4 = 0x01000000
NTLMSSP_NEGOTIATE_TARGET_INFO = 0x00800000
NTLMSSP_REQUEST_NON_NT_SESSION_KEY = 0x00400000
NTLMSSP_RESERVED_R5 = 0x00200000
NTLMSSP_NEGOTIATE_IDENTITY = 0x00100000
NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY = 0x00080000
NTLMSSP_RESERVED_R6 = 0x00040000
NTLMSSP_TARGET_TYPE_SERVER = 0x00020000
NTLMSSP_TARGET_TYPE_DOMAIN = 0x00010000
NTLMSSP_NEGOTIATE_ALWAYS_SIGN = 0x00008000
NTLMSSP_RESERVED_R7 = 0x00004000
NTLMSSP_NEGOTIATE_OEM_WORKSTATION_SUPPLIED = 0x00002000
NTLMSSP_NEGOTIATE_OEM_DOMAIN_SUPPLIED = 0x00001000
NTLMSSP_ANOYNMOUS = 0x00000800
NTLMSSP_RESERVED_R8 = 0x00000400
NTLMSSP_NEGOTIATE_NTLM = 0x00000200
NTLMSSP_RESERVED_R9 = 0x00000100
NTLMSSP_NEGOTIATE_LM_KEY = 0x00000080
NTLMSSP_NEGOTIATE_DATAGRAM = 0x00000040
NTLMSSP_NEGOTIATE_SEAL = 0x00000020
NTLMSSP_NEGOTIATE_SIGN = 0x00000010
NTLMSSP_RESERVED_R10 = 0x00000008
NTLMSSP_REQUEST_TARGET = 0x00000004
NTLMSSP_NEGOTIATE_OEM = 0x00000002
NTLMSSP_NEGOTIATE_UNICODE = 0x00000001
class SignSealConstants(object):
# Magic Contants used to get the signing and sealing key for Extended Session Security
CLIENT_SIGNING = b"session key to client-to-server signing key magic constant\0"
SERVER_SIGNING = b"session key to server-to-client signing key magic constant\0"
CLIENT_SEALING = b"session key to client-to-server sealing key magic constant\0"
SERVER_SEALING = b"session key to server-to-client sealing key magic constant\0"

View File

@ -1,88 +0,0 @@
# This file is part of 'NTLM Authorization Proxy Server' http://sourceforge.net/projects/ntlmaps/
# Copyright 2001 Dmitry A. Rozmanov <dima@xenon.spb.ru>
#
# This library is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
import logging
import six
from ntlm_auth import des_c
log = logging.getLogger(__name__)
class DES:
des_c_obj = None
def __init__(self, key_str):
k = str_to_key56(key_str)
k = key56_to_key64(k)
key_str = b''
for i in k:
key_str += six.int2byte(i & 0xFF)
self.des_c_obj = des_c.DES(key_str)
def encrypt(self, plain_text):
return self.des_c_obj.encrypt(plain_text)
def decrypt(self, crypted_text):
return self.des_c_obj.decrypt(crypted_text)
DESException = 'DESException'
def str_to_key56(key_str):
if not type(key_str) == six.binary_type:
# TODO rsanders high - figure out how to make this not necessary
key_str = key_str.encode('ascii')
if len(key_str) < 7:
key_str = key_str + b'\000\000\000\000\000\000\000'[:(7 - len(key_str))]
key_56 = []
for i in six.iterbytes(key_str[:7]):
key_56.append(i)
return key_56
def key56_to_key64(key_56):
key = []
for i in range(8):
key.append(0)
key[0] = key_56[0]
key[1] = ((key_56[0] << 7) & 0xFF) | (key_56[1] >> 1)
key[2] = ((key_56[1] << 6) & 0xFF) | (key_56[2] >> 2)
key[3] = ((key_56[2] << 5) & 0xFF) | (key_56[3] >> 3)
key[4] = ((key_56[3] << 4) & 0xFF) | (key_56[4] >> 4)
key[5] = ((key_56[4] << 3) & 0xFF) | (key_56[5] >> 5)
key[6] = ((key_56[5] << 2) & 0xFF) | (key_56[6] >> 6)
key[7] = (key_56[6] << 1) & 0xFF
key = set_key_odd_parity(key)
return key
def set_key_odd_parity(key):
for i in range(len(key)):
for k in range(7):
bit = 0
t = key[i] >> k
bit = (t ^ bit) & 0x1
key[i] = (key[i] & 0xFE) | bit
return key

View File

@ -1,254 +0,0 @@
# This file is part of 'NTLM Authorization Proxy Server' http://sourceforge.net/projects/ntlmaps/
# Copyright 2001 Dmitry A. Rozmanov <dima@xenon.spb.ru>
#
# This library is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
import six
from ntlm_auth.U32 import U32
from ntlm_auth.des_data import des_SPtrans, des_skb
def c2l(c):
"char[4] to unsigned long"
l = U32(c[0])
l = l | (U32(c[1]) << 8)
l = l | (U32(c[2]) << 16)
l = l | (U32(c[3]) << 24)
return l
def l2c(l):
"unsigned long to char[4]"
c = []
c.append(int(l & U32(0xFF)))
c.append(int((l >> 8) & U32(0xFF)))
c.append(int((l >> 16) & U32(0xFF)))
c.append(int((l >> 24) & U32(0xFF)))
return c
def D_ENCRYPT(tup, u, t, s):
L, R, S = tup
# print 'LRS1', L, R, S, u, t, '-->',
u = (R ^ s[S])
t = R ^ s[S + 1]
t = ((t >> 4) + (t << 28))
L = L ^ (des_SPtrans[1][int((t) & U32(0x3f))] |
des_SPtrans[3][int((t >> 8) & U32(0x3f))] |
des_SPtrans[5][int((t >> 16) & U32(0x3f))] |
des_SPtrans[7][int((t >> 24) & U32(0x3f))] |
des_SPtrans[0][int((u) & U32(0x3f))] |
des_SPtrans[2][int((u >> 8) & U32(0x3f))] |
des_SPtrans[4][int((u >> 16) & U32(0x3f))] |
des_SPtrans[6][int((u >> 24) & U32(0x3f))])
# print 'LRS:', L, R, S, u, t
return (L, R, S), u, t, s
def PERM_OP(tup, n, m):
"tup - (a, b, t)"
a, b, t = tup
t = ((a >> n) ^ b) & m
b = b ^ t
a = a ^ (t << n)
return (a, b, t)
def HPERM_OP(tup, n, m):
"tup - (a, t)"
a, t = tup
t = ((a << (16 - n)) ^ a) & m
a = a ^ t ^ (t >> (16 - n))
return a, t
shifts2 = [0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0]
class DES:
KeySched = None # des_key_schedule
def __init__(self, key_str):
self.KeySched = des_set_key(key_str)
def decrypt(self, str):
# block - UChar[]
block = []
for i in six.iterbytes(str):
block.append(i)
# print block
block = des_ecb_encrypt(block, self.KeySched, 0)
res = b''
for i in block:
res = res + six.int2byte(i)
return res
def encrypt(self, plaintext):
# block - UChar[]
block = []
for i in plaintext:
block.append(i)
block = des_ecb_encrypt(block, self.KeySched, 1)
res = b''
for i in block:
res += six.int2byte(i)
return res
def des_encript(input, ks, encrypt):
# input - U32[]
# output - U32[]
# ks - des_key_shedule - U32[2][16]
# encrypt - int
# l, r, t, u - U32
# i - int
# s - U32[]
l = input[0]
r = input[1]
t = U32(0)
u = U32(0)
r, l, t = PERM_OP((r, l, t), 4, U32(0x0f0f0f0f))
l, r, t = PERM_OP((l, r, t), 16, U32(0x0000ffff))
r, l, t = PERM_OP((r, l, t), 2, U32(0x33333333))
l, r, t = PERM_OP((l, r, t), 8, U32(0x00ff00ff))
r, l, t = PERM_OP((r, l, t), 1, U32(0x55555555))
t = (r << 1) | (r >> 31)
r = (l << 1) | (l >> 31)
l = t
s = ks # ???????????????
# print l, r
if encrypt:
for i in range(0, 32, 4):
rtup, u, t, s = D_ENCRYPT((l, r, i + 0), u, t, s)
l = rtup[0]
r = rtup[1]
rtup, u, t, s = D_ENCRYPT((r, l, i + 2), u, t, s)
r = rtup[0]
l = rtup[1]
else:
for i in range(30, 0, -4):
rtup, u, t, s = D_ENCRYPT((l, r, i - 0), u, t, s)
l = rtup[0]
r = rtup[1]
rtup, u, t, s = D_ENCRYPT((r, l, i - 2), u, t, s)
r = rtup[0]
l = rtup[1]
# print l, r
l = (l >> 1) | (l << 31)
r = (r >> 1) | (r << 31)
r, l, t = PERM_OP((r, l, t), 1, U32(0x55555555))
l, r, t = PERM_OP((l, r, t), 8, U32(0x00ff00ff))
r, l, t = PERM_OP((r, l, t), 2, U32(0x33333333))
l, r, t = PERM_OP((l, r, t), 16, U32(0x0000ffff))
r, l, t = PERM_OP((r, l, t), 4, U32(0x0f0f0f0f))
output = [l]
output.append(r)
l, r, t, u = U32(0), U32(0), U32(0), U32(0)
return output
def des_ecb_encrypt(input, ks, encrypt):
# input - des_cblock - UChar[8]
# output - des_cblock - UChar[8]
# ks - des_key_shedule - U32[2][16]
# encrypt - int
# print input
l0 = c2l(input[0:4])
l1 = c2l(input[4:8])
ll = [l0]
ll.append(l1)
# print ll
ll = des_encript(ll, ks, encrypt)
# print ll
l0 = ll[0]
l1 = ll[1]
output = l2c(l0)
output = output + l2c(l1)
# print output
l0, l1, ll[0], ll[1] = U32(0), U32(0), U32(0), U32(0)
return output
def des_set_key(key):
# key - des_cblock - UChar[8]
# schedule - des_key_schedule
# register unsigned long c,d,t,s;
# register unsigned char *in;
# register unsigned long *k;
# register int i;
# k = schedule
# in = key
k = []
c = c2l(key[0:4])
d = c2l(key[4:8])
t = U32(0)
d, c, t = PERM_OP((d, c, t), 4, U32(0x0f0f0f0f))
c, t = HPERM_OP((c, t), -2, U32(0xcccc0000))
d, t = HPERM_OP((d, t), -2, U32(0xcccc0000))
d, c, t = PERM_OP((d, c, t), 1, U32(0x55555555))
c, d, t = PERM_OP((c, d, t), 8, U32(0x00ff00ff))
d, c, t = PERM_OP((d, c, t), 1, U32(0x55555555))
d = (((d & U32(0x000000ff)) << 16) | (d & U32(0x0000ff00)) | ((d & U32(0x00ff0000)) >> 16) | (
(c & U32(0xf0000000)) >> 4))
c = c & U32(0x0fffffff)
for i in range(16):
if (shifts2[i]):
c = ((c >> 2) | (c << 26))
d = ((d >> 2) | (d << 26))
else:
c = ((c >> 1) | (c << 27))
d = ((d >> 1) | (d << 27))
c = c & U32(0x0fffffff)
d = d & U32(0x0fffffff)
s = des_skb[0][int((c) & U32(0x3f))] | \
des_skb[1][int(((c >> 6) & U32(0x03)) | ((c >> 7) & U32(0x3c)))] | \
des_skb[2][int(((c >> 13) & U32(0x0f)) | ((c >> 14) & U32(0x30)))] | \
des_skb[3][int(((c >> 20) & U32(0x01)) | ((c >> 21) & U32(0x06)) | ((c >> 22) & U32(0x38)))]
t = des_skb[4][int((d) & U32(0x3f))] | \
des_skb[5][int(((d >> 7) & U32(0x03)) | ((d >> 8) & U32(0x3c)))] | \
des_skb[6][int((d >> 15) & U32(0x3f))] | \
des_skb[7][int(((d >> 21) & U32(0x0f)) | ((d >> 22) & U32(0x30)))]
# print s, t
k.append(((t << 16) | (s & U32(0x0000ffff))) & U32(0xffffffff))
s = ((s >> 16) | (t & U32(0xffff0000)))
s = (s << 4) | (s >> 28)
k.append(s & U32(0xffffffff))
schedule = k
return schedule

View File

@ -1,348 +0,0 @@
# This file is part of 'NTLM Authorization Proxy Server' http://sourceforge.net/projects/ntlmaps/
# Copyright 2001 Dmitry A. Rozmanov <dima@xenon.spb.ru>
#
# This library is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
from ntlm_auth.U32 import U32
# static unsigned long des_SPtrans[8][64]={
des_SPtrans = \
[
# nibble 0
[
U32(0x00820200), U32(0x00020000), U32(0x80800000), U32(0x80820200),
U32(0x00800000), U32(0x80020200), U32(0x80020000), U32(0x80800000),
U32(0x80020200), U32(0x00820200), U32(0x00820000), U32(0x80000200),
U32(0x80800200), U32(0x00800000), U32(0x00000000), U32(0x80020000),
U32(0x00020000), U32(0x80000000), U32(0x00800200), U32(0x00020200),
U32(0x80820200), U32(0x00820000), U32(0x80000200), U32(0x00800200),
U32(0x80000000), U32(0x00000200), U32(0x00020200), U32(0x80820000),
U32(0x00000200), U32(0x80800200), U32(0x80820000), U32(0x00000000),
U32(0x00000000), U32(0x80820200), U32(0x00800200), U32(0x80020000),
U32(0x00820200), U32(0x00020000), U32(0x80000200), U32(0x00800200),
U32(0x80820000), U32(0x00000200), U32(0x00020200), U32(0x80800000),
U32(0x80020200), U32(0x80000000), U32(0x80800000), U32(0x00820000),
U32(0x80820200), U32(0x00020200), U32(0x00820000), U32(0x80800200),
U32(0x00800000), U32(0x80000200), U32(0x80020000), U32(0x00000000),
U32(0x00020000), U32(0x00800000), U32(0x80800200), U32(0x00820200),
U32(0x80000000), U32(0x80820000), U32(0x00000200), U32(0x80020200),
],
# nibble 1
[
U32(0x10042004), U32(0x00000000), U32(0x00042000), U32(0x10040000),
U32(0x10000004), U32(0x00002004), U32(0x10002000), U32(0x00042000),
U32(0x00002000), U32(0x10040004), U32(0x00000004), U32(0x10002000),
U32(0x00040004), U32(0x10042000), U32(0x10040000), U32(0x00000004),
U32(0x00040000), U32(0x10002004), U32(0x10040004), U32(0x00002000),
U32(0x00042004), U32(0x10000000), U32(0x00000000), U32(0x00040004),
U32(0x10002004), U32(0x00042004), U32(0x10042000), U32(0x10000004),
U32(0x10000000), U32(0x00040000), U32(0x00002004), U32(0x10042004),
U32(0x00040004), U32(0x10042000), U32(0x10002000), U32(0x00042004),
U32(0x10042004), U32(0x00040004), U32(0x10000004), U32(0x00000000),
U32(0x10000000), U32(0x00002004), U32(0x00040000), U32(0x10040004),
U32(0x00002000), U32(0x10000000), U32(0x00042004), U32(0x10002004),
U32(0x10042000), U32(0x00002000), U32(0x00000000), U32(0x10000004),
U32(0x00000004), U32(0x10042004), U32(0x00042000), U32(0x10040000),
U32(0x10040004), U32(0x00040000), U32(0x00002004), U32(0x10002000),
U32(0x10002004), U32(0x00000004), U32(0x10040000), U32(0x00042000),
],
# nibble 2
[
U32(0x41000000), U32(0x01010040), U32(0x00000040), U32(0x41000040),
U32(0x40010000), U32(0x01000000), U32(0x41000040), U32(0x00010040),
U32(0x01000040), U32(0x00010000), U32(0x01010000), U32(0x40000000),
U32(0x41010040), U32(0x40000040), U32(0x40000000), U32(0x41010000),
U32(0x00000000), U32(0x40010000), U32(0x01010040), U32(0x00000040),
U32(0x40000040), U32(0x41010040), U32(0x00010000), U32(0x41000000),
U32(0x41010000), U32(0x01000040), U32(0x40010040), U32(0x01010000),
U32(0x00010040), U32(0x00000000), U32(0x01000000), U32(0x40010040),
U32(0x01010040), U32(0x00000040), U32(0x40000000), U32(0x00010000),
U32(0x40000040), U32(0x40010000), U32(0x01010000), U32(0x41000040),
U32(0x00000000), U32(0x01010040), U32(0x00010040), U32(0x41010000),
U32(0x40010000), U32(0x01000000), U32(0x41010040), U32(0x40000000),
U32(0x40010040), U32(0x41000000), U32(0x01000000), U32(0x41010040),
U32(0x00010000), U32(0x01000040), U32(0x41000040), U32(0x00010040),
U32(0x01000040), U32(0x00000000), U32(0x41010000), U32(0x40000040),
U32(0x41000000), U32(0x40010040), U32(0x00000040), U32(0x01010000),
],
# nibble 3
[
U32(0x00100402), U32(0x04000400), U32(0x00000002), U32(0x04100402),
U32(0x00000000), U32(0x04100000), U32(0x04000402), U32(0x00100002),
U32(0x04100400), U32(0x04000002), U32(0x04000000), U32(0x00000402),
U32(0x04000002), U32(0x00100402), U32(0x00100000), U32(0x04000000),
U32(0x04100002), U32(0x00100400), U32(0x00000400), U32(0x00000002),
U32(0x00100400), U32(0x04000402), U32(0x04100000), U32(0x00000400),
U32(0x00000402), U32(0x00000000), U32(0x00100002), U32(0x04100400),
U32(0x04000400), U32(0x04100002), U32(0x04100402), U32(0x00100000),
U32(0x04100002), U32(0x00000402), U32(0x00100000), U32(0x04000002),
U32(0x00100400), U32(0x04000400), U32(0x00000002), U32(0x04100000),
U32(0x04000402), U32(0x00000000), U32(0x00000400), U32(0x00100002),
U32(0x00000000), U32(0x04100002), U32(0x04100400), U32(0x00000400),
U32(0x04000000), U32(0x04100402), U32(0x00100402), U32(0x00100000),
U32(0x04100402), U32(0x00000002), U32(0x04000400), U32(0x00100402),
U32(0x00100002), U32(0x00100400), U32(0x04100000), U32(0x04000402),
U32(0x00000402), U32(0x04000000), U32(0x04000002), U32(0x04100400),
],
# nibble 4
[
U32(0x02000000), U32(0x00004000), U32(0x00000100), U32(0x02004108),
U32(0x02004008), U32(0x02000100), U32(0x00004108), U32(0x02004000),
U32(0x00004000), U32(0x00000008), U32(0x02000008), U32(0x00004100),
U32(0x02000108), U32(0x02004008), U32(0x02004100), U32(0x00000000),
U32(0x00004100), U32(0x02000000), U32(0x00004008), U32(0x00000108),
U32(0x02000100), U32(0x00004108), U32(0x00000000), U32(0x02000008),
U32(0x00000008), U32(0x02000108), U32(0x02004108), U32(0x00004008),
U32(0x02004000), U32(0x00000100), U32(0x00000108), U32(0x02004100),
U32(0x02004100), U32(0x02000108), U32(0x00004008), U32(0x02004000),
U32(0x00004000), U32(0x00000008), U32(0x02000008), U32(0x02000100),
U32(0x02000000), U32(0x00004100), U32(0x02004108), U32(0x00000000),
U32(0x00004108), U32(0x02000000), U32(0x00000100), U32(0x00004008),
U32(0x02000108), U32(0x00000100), U32(0x00000000), U32(0x02004108),
U32(0x02004008), U32(0x02004100), U32(0x00000108), U32(0x00004000),
U32(0x00004100), U32(0x02004008), U32(0x02000100), U32(0x00000108),
U32(0x00000008), U32(0x00004108), U32(0x02004000), U32(0x02000008),
],
# nibble 5
[
U32(0x20000010), U32(0x00080010), U32(0x00000000), U32(0x20080800),
U32(0x00080010), U32(0x00000800), U32(0x20000810), U32(0x00080000),
U32(0x00000810), U32(0x20080810), U32(0x00080800), U32(0x20000000),
U32(0x20000800), U32(0x20000010), U32(0x20080000), U32(0x00080810),
U32(0x00080000), U32(0x20000810), U32(0x20080010), U32(0x00000000),
U32(0x00000800), U32(0x00000010), U32(0x20080800), U32(0x20080010),
U32(0x20080810), U32(0x20080000), U32(0x20000000), U32(0x00000810),
U32(0x00000010), U32(0x00080800), U32(0x00080810), U32(0x20000800),
U32(0x00000810), U32(0x20000000), U32(0x20000800), U32(0x00080810),
U32(0x20080800), U32(0x00080010), U32(0x00000000), U32(0x20000800),
U32(0x20000000), U32(0x00000800), U32(0x20080010), U32(0x00080000),
U32(0x00080010), U32(0x20080810), U32(0x00080800), U32(0x00000010),
U32(0x20080810), U32(0x00080800), U32(0x00080000), U32(0x20000810),
U32(0x20000010), U32(0x20080000), U32(0x00080810), U32(0x00000000),
U32(0x00000800), U32(0x20000010), U32(0x20000810), U32(0x20080800),
U32(0x20080000), U32(0x00000810), U32(0x00000010), U32(0x20080010),
],
# nibble 6
[
U32(0x00001000), U32(0x00000080), U32(0x00400080), U32(0x00400001),
U32(0x00401081), U32(0x00001001), U32(0x00001080), U32(0x00000000),
U32(0x00400000), U32(0x00400081), U32(0x00000081), U32(0x00401000),
U32(0x00000001), U32(0x00401080), U32(0x00401000), U32(0x00000081),
U32(0x00400081), U32(0x00001000), U32(0x00001001), U32(0x00401081),
U32(0x00000000), U32(0x00400080), U32(0x00400001), U32(0x00001080),
U32(0x00401001), U32(0x00001081), U32(0x00401080), U32(0x00000001),
U32(0x00001081), U32(0x00401001), U32(0x00000080), U32(0x00400000),
U32(0x00001081), U32(0x00401000), U32(0x00401001), U32(0x00000081),
U32(0x00001000), U32(0x00000080), U32(0x00400000), U32(0x00401001),
U32(0x00400081), U32(0x00001081), U32(0x00001080), U32(0x00000000),
U32(0x00000080), U32(0x00400001), U32(0x00000001), U32(0x00400080),
U32(0x00000000), U32(0x00400081), U32(0x00400080), U32(0x00001080),
U32(0x00000081), U32(0x00001000), U32(0x00401081), U32(0x00400000),
U32(0x00401080), U32(0x00000001), U32(0x00001001), U32(0x00401081),
U32(0x00400001), U32(0x00401080), U32(0x00401000), U32(0x00001001),
],
# nibble 7
[
U32(0x08200020), U32(0x08208000), U32(0x00008020), U32(0x00000000),
U32(0x08008000), U32(0x00200020), U32(0x08200000), U32(0x08208020),
U32(0x00000020), U32(0x08000000), U32(0x00208000), U32(0x00008020),
U32(0x00208020), U32(0x08008020), U32(0x08000020), U32(0x08200000),
U32(0x00008000), U32(0x00208020), U32(0x00200020), U32(0x08008000),
U32(0x08208020), U32(0x08000020), U32(0x00000000), U32(0x00208000),
U32(0x08000000), U32(0x00200000), U32(0x08008020), U32(0x08200020),
U32(0x00200000), U32(0x00008000), U32(0x08208000), U32(0x00000020),
U32(0x00200000), U32(0x00008000), U32(0x08000020), U32(0x08208020),
U32(0x00008020), U32(0x08000000), U32(0x00000000), U32(0x00208000),
U32(0x08200020), U32(0x08008020), U32(0x08008000), U32(0x00200020),
U32(0x08208000), U32(0x00000020), U32(0x00200020), U32(0x08008000),
U32(0x08208020), U32(0x00200000), U32(0x08200000), U32(0x08000020),
U32(0x00208000), U32(0x00008020), U32(0x08008020), U32(0x08200000),
U32(0x00000020), U32(0x08208000), U32(0x00208020), U32(0x00000000),
U32(0x08000000), U32(0x08200020), U32(0x00008000), U32(0x00208020),
],
]
# static unsigned long des_skb[8][64]={
des_skb = \
[
# for C bits (numbered as per FIPS 46) 1 2 3 4 5 6
[
U32(0x00000000), U32(0x00000010), U32(0x20000000), U32(0x20000010),
U32(0x00010000), U32(0x00010010), U32(0x20010000), U32(0x20010010),
U32(0x00000800), U32(0x00000810), U32(0x20000800), U32(0x20000810),
U32(0x00010800), U32(0x00010810), U32(0x20010800), U32(0x20010810),
U32(0x00000020), U32(0x00000030), U32(0x20000020), U32(0x20000030),
U32(0x00010020), U32(0x00010030), U32(0x20010020), U32(0x20010030),
U32(0x00000820), U32(0x00000830), U32(0x20000820), U32(0x20000830),
U32(0x00010820), U32(0x00010830), U32(0x20010820), U32(0x20010830),
U32(0x00080000), U32(0x00080010), U32(0x20080000), U32(0x20080010),
U32(0x00090000), U32(0x00090010), U32(0x20090000), U32(0x20090010),
U32(0x00080800), U32(0x00080810), U32(0x20080800), U32(0x20080810),
U32(0x00090800), U32(0x00090810), U32(0x20090800), U32(0x20090810),
U32(0x00080020), U32(0x00080030), U32(0x20080020), U32(0x20080030),
U32(0x00090020), U32(0x00090030), U32(0x20090020), U32(0x20090030),
U32(0x00080820), U32(0x00080830), U32(0x20080820), U32(0x20080830),
U32(0x00090820), U32(0x00090830), U32(0x20090820), U32(0x20090830),
],
# for C bits (numbered as per FIPS 46) 7 8 10 11 12 13
[
U32(0x00000000), U32(0x02000000), U32(0x00002000), U32(0x02002000),
U32(0x00200000), U32(0x02200000), U32(0x00202000), U32(0x02202000),
U32(0x00000004), U32(0x02000004), U32(0x00002004), U32(0x02002004),
U32(0x00200004), U32(0x02200004), U32(0x00202004), U32(0x02202004),
U32(0x00000400), U32(0x02000400), U32(0x00002400), U32(0x02002400),
U32(0x00200400), U32(0x02200400), U32(0x00202400), U32(0x02202400),
U32(0x00000404), U32(0x02000404), U32(0x00002404), U32(0x02002404),
U32(0x00200404), U32(0x02200404), U32(0x00202404), U32(0x02202404),
U32(0x10000000), U32(0x12000000), U32(0x10002000), U32(0x12002000),
U32(0x10200000), U32(0x12200000), U32(0x10202000), U32(0x12202000),
U32(0x10000004), U32(0x12000004), U32(0x10002004), U32(0x12002004),
U32(0x10200004), U32(0x12200004), U32(0x10202004), U32(0x12202004),
U32(0x10000400), U32(0x12000400), U32(0x10002400), U32(0x12002400),
U32(0x10200400), U32(0x12200400), U32(0x10202400), U32(0x12202400),
U32(0x10000404), U32(0x12000404), U32(0x10002404), U32(0x12002404),
U32(0x10200404), U32(0x12200404), U32(0x10202404), U32(0x12202404),
],
# for C bits (numbered as per FIPS 46) 14 15 16 17 19 20
[
U32(0x00000000), U32(0x00000001), U32(0x00040000), U32(0x00040001),
U32(0x01000000), U32(0x01000001), U32(0x01040000), U32(0x01040001),
U32(0x00000002), U32(0x00000003), U32(0x00040002), U32(0x00040003),
U32(0x01000002), U32(0x01000003), U32(0x01040002), U32(0x01040003),
U32(0x00000200), U32(0x00000201), U32(0x00040200), U32(0x00040201),
U32(0x01000200), U32(0x01000201), U32(0x01040200), U32(0x01040201),
U32(0x00000202), U32(0x00000203), U32(0x00040202), U32(0x00040203),
U32(0x01000202), U32(0x01000203), U32(0x01040202), U32(0x01040203),
U32(0x08000000), U32(0x08000001), U32(0x08040000), U32(0x08040001),
U32(0x09000000), U32(0x09000001), U32(0x09040000), U32(0x09040001),
U32(0x08000002), U32(0x08000003), U32(0x08040002), U32(0x08040003),
U32(0x09000002), U32(0x09000003), U32(0x09040002), U32(0x09040003),
U32(0x08000200), U32(0x08000201), U32(0x08040200), U32(0x08040201),
U32(0x09000200), U32(0x09000201), U32(0x09040200), U32(0x09040201),
U32(0x08000202), U32(0x08000203), U32(0x08040202), U32(0x08040203),
U32(0x09000202), U32(0x09000203), U32(0x09040202), U32(0x09040203),
],
# for C bits (numbered as per FIPS 46) 21 23 24 26 27 28
[
U32(0x00000000), U32(0x00100000), U32(0x00000100), U32(0x00100100),
U32(0x00000008), U32(0x00100008), U32(0x00000108), U32(0x00100108),
U32(0x00001000), U32(0x00101000), U32(0x00001100), U32(0x00101100),
U32(0x00001008), U32(0x00101008), U32(0x00001108), U32(0x00101108),
U32(0x04000000), U32(0x04100000), U32(0x04000100), U32(0x04100100),
U32(0x04000008), U32(0x04100008), U32(0x04000108), U32(0x04100108),
U32(0x04001000), U32(0x04101000), U32(0x04001100), U32(0x04101100),
U32(0x04001008), U32(0x04101008), U32(0x04001108), U32(0x04101108),
U32(0x00020000), U32(0x00120000), U32(0x00020100), U32(0x00120100),
U32(0x00020008), U32(0x00120008), U32(0x00020108), U32(0x00120108),
U32(0x00021000), U32(0x00121000), U32(0x00021100), U32(0x00121100),
U32(0x00021008), U32(0x00121008), U32(0x00021108), U32(0x00121108),
U32(0x04020000), U32(0x04120000), U32(0x04020100), U32(0x04120100),
U32(0x04020008), U32(0x04120008), U32(0x04020108), U32(0x04120108),
U32(0x04021000), U32(0x04121000), U32(0x04021100), U32(0x04121100),
U32(0x04021008), U32(0x04121008), U32(0x04021108), U32(0x04121108),
],
# for D bits (numbered as per FIPS 46) 1 2 3 4 5 6
[
U32(0x00000000), U32(0x10000000), U32(0x00010000), U32(0x10010000),
U32(0x00000004), U32(0x10000004), U32(0x00010004), U32(0x10010004),
U32(0x20000000), U32(0x30000000), U32(0x20010000), U32(0x30010000),
U32(0x20000004), U32(0x30000004), U32(0x20010004), U32(0x30010004),
U32(0x00100000), U32(0x10100000), U32(0x00110000), U32(0x10110000),
U32(0x00100004), U32(0x10100004), U32(0x00110004), U32(0x10110004),
U32(0x20100000), U32(0x30100000), U32(0x20110000), U32(0x30110000),
U32(0x20100004), U32(0x30100004), U32(0x20110004), U32(0x30110004),
U32(0x00001000), U32(0x10001000), U32(0x00011000), U32(0x10011000),
U32(0x00001004), U32(0x10001004), U32(0x00011004), U32(0x10011004),
U32(0x20001000), U32(0x30001000), U32(0x20011000), U32(0x30011000),
U32(0x20001004), U32(0x30001004), U32(0x20011004), U32(0x30011004),
U32(0x00101000), U32(0x10101000), U32(0x00111000), U32(0x10111000),
U32(0x00101004), U32(0x10101004), U32(0x00111004), U32(0x10111004),
U32(0x20101000), U32(0x30101000), U32(0x20111000), U32(0x30111000),
U32(0x20101004), U32(0x30101004), U32(0x20111004), U32(0x30111004),
],
# for D bits (numbered as per FIPS 46) 8 9 11 12 13 14
[
U32(0x00000000), U32(0x08000000), U32(0x00000008), U32(0x08000008),
U32(0x00000400), U32(0x08000400), U32(0x00000408), U32(0x08000408),
U32(0x00020000), U32(0x08020000), U32(0x00020008), U32(0x08020008),
U32(0x00020400), U32(0x08020400), U32(0x00020408), U32(0x08020408),
U32(0x00000001), U32(0x08000001), U32(0x00000009), U32(0x08000009),
U32(0x00000401), U32(0x08000401), U32(0x00000409), U32(0x08000409),
U32(0x00020001), U32(0x08020001), U32(0x00020009), U32(0x08020009),
U32(0x00020401), U32(0x08020401), U32(0x00020409), U32(0x08020409),
U32(0x02000000), U32(0x0A000000), U32(0x02000008), U32(0x0A000008),
U32(0x02000400), U32(0x0A000400), U32(0x02000408), U32(0x0A000408),
U32(0x02020000), U32(0x0A020000), U32(0x02020008), U32(0x0A020008),
U32(0x02020400), U32(0x0A020400), U32(0x02020408), U32(0x0A020408),
U32(0x02000001), U32(0x0A000001), U32(0x02000009), U32(0x0A000009),
U32(0x02000401), U32(0x0A000401), U32(0x02000409), U32(0x0A000409),
U32(0x02020001), U32(0x0A020001), U32(0x02020009), U32(0x0A020009),
U32(0x02020401), U32(0x0A020401), U32(0x02020409), U32(0x0A020409),
],
# for D bits (numbered as per FIPS 46) 16 17 18 19 20 21
[
U32(0x00000000), U32(0x00000100), U32(0x00080000), U32(0x00080100),
U32(0x01000000), U32(0x01000100), U32(0x01080000), U32(0x01080100),
U32(0x00000010), U32(0x00000110), U32(0x00080010), U32(0x00080110),
U32(0x01000010), U32(0x01000110), U32(0x01080010), U32(0x01080110),
U32(0x00200000), U32(0x00200100), U32(0x00280000), U32(0x00280100),
U32(0x01200000), U32(0x01200100), U32(0x01280000), U32(0x01280100),
U32(0x00200010), U32(0x00200110), U32(0x00280010), U32(0x00280110),
U32(0x01200010), U32(0x01200110), U32(0x01280010), U32(0x01280110),
U32(0x00000200), U32(0x00000300), U32(0x00080200), U32(0x00080300),
U32(0x01000200), U32(0x01000300), U32(0x01080200), U32(0x01080300),
U32(0x00000210), U32(0x00000310), U32(0x00080210), U32(0x00080310),
U32(0x01000210), U32(0x01000310), U32(0x01080210), U32(0x01080310),
U32(0x00200200), U32(0x00200300), U32(0x00280200), U32(0x00280300),
U32(0x01200200), U32(0x01200300), U32(0x01280200), U32(0x01280300),
U32(0x00200210), U32(0x00200310), U32(0x00280210), U32(0x00280310),
U32(0x01200210), U32(0x01200310), U32(0x01280210), U32(0x01280310),
],
# for D bits (numbered as per FIPS 46) 22 23 24 25 27 28
[
U32(0x00000000), U32(0x04000000), U32(0x00040000), U32(0x04040000),
U32(0x00000002), U32(0x04000002), U32(0x00040002), U32(0x04040002),
U32(0x00002000), U32(0x04002000), U32(0x00042000), U32(0x04042000),
U32(0x00002002), U32(0x04002002), U32(0x00042002), U32(0x04042002),
U32(0x00000020), U32(0x04000020), U32(0x00040020), U32(0x04040020),
U32(0x00000022), U32(0x04000022), U32(0x00040022), U32(0x04040022),
U32(0x00002020), U32(0x04002020), U32(0x00042020), U32(0x04042020),
U32(0x00002022), U32(0x04002022), U32(0x00042022), U32(0x04042022),
U32(0x00000800), U32(0x04000800), U32(0x00040800), U32(0x04040800),
U32(0x00000802), U32(0x04000802), U32(0x00040802), U32(0x04040802),
U32(0x00002800), U32(0x04002800), U32(0x00042800), U32(0x04042800),
U32(0x00002802), U32(0x04002802), U32(0x00042802), U32(0x04042802),
U32(0x00000820), U32(0x04000820), U32(0x00040820), U32(0x04040820),
U32(0x00000822), U32(0x04000822), U32(0x00040822), U32(0x04040822),
U32(0x00002820), U32(0x04002820), U32(0x00042820), U32(0x04042820),
U32(0x00002822), U32(0x04002822), U32(0x00042822), U32(0x04042822),
]
]

View File

@ -1,67 +0,0 @@
# This library is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
import struct
"""
This is not the easiest structure to understand, ultimately this is a set structure
as defined by Microsoft. Channel Binding Tokens set the SHA256 hash of the server
certificate to the application_data field and then ultimately creates the MD5 hash
to include in the NTLM auth from there. This class is just designed to create the
bindings structure which is then used by compute_response.py to do the rest of the
work.
For more infor on how this works and how it is derived, this is a great link;
https://blogs.msdn.microsoft.com/openspecification/2013/03/26/ntlm-and-channel-binding-hash-aka-extended-protection-for-authentication/
"""
class GssChannelBindingsStruct(object):
INITIATOR_ADDTYPE = 'initiator_addtype'
INITIATOR_ADDRESS_LENGTH = 'initiator_address_length'
ACCEPTOR_ADDRTYPE = 'acceptor_addrtype'
ACCEPTOR_ADDRESS_LENGTH = 'acceptor_address_length'
APPLICATION_DATA_LENGTH = 'application_data_length'
INITIATOR_ADDRESS = 'initiator_address'
ACCEPTOR_ADDRESS = 'acceptor_address'
APPLICATION_DATA = 'application_data'
def __init__(self):
self.fields = {}
self.fields[self.INITIATOR_ADDTYPE] = 0
self.fields[self.INITIATOR_ADDRESS_LENGTH] = 0
self.fields[self.ACCEPTOR_ADDRTYPE] = 0
self.fields[self.ACCEPTOR_ADDRESS_LENGTH] = 0
self.fields[self.APPLICATION_DATA_LENGTH] = 0
self.fields[self.INITIATOR_ADDRESS] = b''
self.fields[self.ACCEPTOR_ADDRESS] = b''
self.fields[self.APPLICATION_DATA] = b''
def __setitem__(self, key, value):
self.fields[key] = value
def get_data(self):
# Set the lengths of each len field in case they have changed
self.fields[self.INITIATOR_ADDRESS_LENGTH] = len(self.fields[self.INITIATOR_ADDRESS])
self.fields[self.ACCEPTOR_ADDRESS_LENGTH] = len(self.fields[self.ACCEPTOR_ADDRESS])
self.fields[self.APPLICATION_DATA_LENGTH] = len(self.fields[self.APPLICATION_DATA])
# Add all the values together to create the gss_channel_bindings_struct
data = struct.pack('<L', self.fields[self.INITIATOR_ADDTYPE]) + \
struct.pack('<L', self.fields[self.INITIATOR_ADDRESS_LENGTH]) + \
self.fields[self.INITIATOR_ADDRESS] + \
struct.pack('<L', self.fields[self.ACCEPTOR_ADDRTYPE]) + \
struct.pack('<L', self.fields[self.ACCEPTOR_ADDRESS_LENGTH]) + \
self.fields[self.ACCEPTOR_ADDRESS] + \
struct.pack('<L', self.fields[self.APPLICATION_DATA_LENGTH]) + \
self.fields[self.APPLICATION_DATA]
return data

View File

@ -1,359 +0,0 @@
# This library is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
import hmac
import os
import struct
from ntlm_auth.compute_response import ComputeResponse
from ntlm_auth.constants import NegotiateFlags, MessageTypes, NTLM_SIGNATURE, AvFlags
from ntlm_auth.rc4 import ARC4
from ntlm_auth.target_info import TargetInfo
class NegotiateMessage(object):
EXPECTED_BODY_LENGTH = 40
"""
[MS-NLMP] v28.0 2016-07-14
2.2.1.1 NEGOTIATE_MESSAGE
The NEGOTIATE_MESSAGE defines an NTLM Negotiate message that is sent from the client to
the server. This message allows the client to specify its supported NTLM options to
the server.
:param negotiate_flags: A NEGOTIATE structure that contains a set of bit flags. These flags are the options the client supports
:param domain_name: The domain name of the user to authenticate with, default is None
:param workstation: The worksation of the client machine, default is None
Attributes:
signature: An 8-byte character array that MUST contain the ASCII string 'NTLMSSP\0'
message_type: A 32-bit unsigned integer that indicates the message type. This field must be set to 0x00000001
negotiate_flags: A NEGOTIATE structure that contains a set of bit flags. These flags are the options the client supports
version: Contains the windows version info of the client. It is used only debugging purposes and are only set when NTLMSSP_NEGOTIATE_VERSION flag is set
domain_name: A byte-array that contains the name of the client authentication domain that MUST Be encoded in the negotiated character set
workstation: A byte-array that contains the name of the client machine that MUST Be encoded in the negotiated character set
"""
def __init__(self, negotiate_flags, domain_name, workstation):
self.signature = NTLM_SIGNATURE
self.message_type = struct.pack('<L', MessageTypes.NTLM_NEGOTIATE)
# Check if the domain_name value is set, if it is, make sure the negotiate_flag is also set
if domain_name is None:
self.domain_name = ''
else:
self.domain_name = domain_name
negotiate_flags |= NegotiateFlags.NTLMSSP_NEGOTIATE_OEM_DOMAIN_SUPPLIED
# Check if the workstation value is set, if it is, make sure the negotiate_flag is also set
if workstation is None:
self.workstation = ''
else:
self.workstation = workstation
negotiate_flags |= NegotiateFlags.NTLMSSP_NEGOTIATE_OEM_WORKSTATION_SUPPLIED
# Set the encoding flag to use OEM, remove UNICODE if set as it isn't support in this message
negotiate_flags -= NegotiateFlags.NTLMSSP_NEGOTIATE_UNICODE
negotiate_flags |= NegotiateFlags.NTLMSSP_NEGOTIATE_OEM
self.domain_name = self.domain_name.encode('ascii')
self.workstation = self.workstation.encode('ascii')
self.version = get_version(negotiate_flags)
self.negotiate_flags = struct.pack('<I', negotiate_flags)
def get_data(self):
payload_offset = self.EXPECTED_BODY_LENGTH
# DomainNameFields - 8 bytes
domain_name_len = struct.pack('<H', len(self.domain_name))
domain_name_max_len = struct.pack('<H', len(self.domain_name))
domain_name_buffer_offset = struct.pack('<I', payload_offset)
payload_offset += len(self.domain_name)
# WorkstationFields - 8 bytes
workstation_len = struct.pack('<H', len(self.workstation))
workstation_max_len = struct.pack('<H', len(self.workstation))
workstation_buffer_offset = struct.pack('<I', payload_offset)
payload_offset += len(self.workstation)
# Payload - variable length
payload = self.domain_name
payload += self.workstation
# Bring the header values together into 1 message
msg1 = self.signature
msg1 += self.message_type
msg1 += self.negotiate_flags
msg1 += domain_name_len
msg1 += domain_name_max_len
msg1 += domain_name_buffer_offset
msg1 += workstation_len
msg1 += workstation_max_len
msg1 += workstation_buffer_offset
msg1 += self.version
assert self.EXPECTED_BODY_LENGTH == len(msg1), "BODY_LENGTH: %d != msg1: %d" % (self.EXPECTED_BODY_LENGTH, len(msg1))
# Adding the payload data to the message
msg1 += payload
return msg1
class ChallengeMessage(object):
"""
[MS-NLMP] v28.0 2016-07-14
2.2.1.2 CHALLENGE_MESSAGE
The CHALLENGE_MESSAGE defines an NTLM challenge message that is sent from the server to
the client. The CHALLENGE_MESSAGE is used by the server to challenge the client to prove
its identity, For connection-oriented requests, the CHALLENGE_MESSAGE generated by the
server is in response to the NEGOTIATE_MESSAGE from the client.
:param msg2: The CHALLENGE_MESSAGE received from the server after sending our NEGOTIATE_MESSAGE. This has
been decoded from a base64 string
Attributes
signature: An 8-byte character array that MUST contain the ASCII string 'NTLMSSP\0'
message_type: A 32-bit unsigned integer that indicates the message type. This field must be set to 0x00000002
negotiate_flags: A NEGOTIATE strucutre that contains a set of bit flags. The server sets flags to indicate options it supports
server_challenge: A 64-bit value that contains the NTLM challenge. The challenge is a 64-bit nonce. Used in the AuthenticateMessage message
reserved: An 8-byte array whose elements MUST be zero when sent and MUST be ignored on receipt
version: When NTLMSSP_NEGOTIATE_VERSION flag is set in negotiate_flags field which contains the windows version info. Used only for debugging purposes
target_name: When NTLMSSP_REQUEST_TARGET is set is a byte array that contains the name of the server authentication realm. In a domain environment this is the domain name not server name
target_info: When NTLMSSP_NEGOTIATE_TARGET_INFO is set is a byte array that contains a sequence of AV_PAIR structures (target_info.py)
"""
def __init__(self, msg2):
self.data = msg2
# Setting the object values from the raw_challenge_message
self.signature = msg2[0:8]
self.message_type = struct.unpack("<I", msg2[8:12])[0]
self.negotiate_flags = struct.unpack("<I", msg2[20:24])[0]
self.server_challenge = msg2[24:32]
self.reserved = msg2[32:40]
if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_VERSION and self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY:
size = len(msg2)
self.version = struct.unpack("<q", msg2[48:56])[0]
else:
self.version = None
if self.negotiate_flags & NegotiateFlags.NTLMSSP_REQUEST_TARGET:
target_name_len = struct.unpack("<H", msg2[12:14])[0]
target_name_max_len = struct.unpack("<H", msg2[14:16])[0]
target_name_buffer_offset = struct.unpack("<I", msg2[16:20])[0]
self.target_name = msg2[target_name_buffer_offset:target_name_buffer_offset + target_name_len]
else:
self.target_name = None
if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_TARGET_INFO:
target_info_len = struct.unpack("<H", msg2[40:42])[0]
target_info_max_len = struct.unpack("<H", msg2[42:44])[0]
target_info_buffer_offset = struct.unpack("<I", msg2[44:48])[0]
target_info_raw = msg2[target_info_buffer_offset:target_info_buffer_offset + target_info_len]
self.target_info = TargetInfo(target_info_raw)
else:
self.target_info = None
# Verify initial integrity of the message, it matches what should be there
assert self.signature == NTLM_SIGNATURE
assert self.message_type == MessageTypes.NTLM_CHALLENGE
def get_data(self):
return self.data
class AuthenticateMessage(object):
EXPECTED_BODY_LENGTH = 72
EXPECTED_BODY_LENGTH_WITH_MIC = 88
"""
[MS-NLMP] v28.0 2016-07-14
2.2.1.3 AUTHENTICATE_MESSAGE
The AUTHENTICATE_MESSAGE defines an NTLM authenticate message that is sent from the
client to the server after the CHALLENGE_MESSAGE is processed by the client.
:param user_name: The user name of the user we are trying to authenticate with
:param password: The password of the user we are trying to authenticate with
:param domain_name: The domain name of the user account we are authenticated with, default is None
:param workstation: The workstation we are using to authenticate with, default is None
:param challenge_message: A ChallengeMessage object that was received from the server after the negotiate_message
:param ntlm_compatibility: The Lan Manager Compatibility Level, used to determine what NTLM auth version to use, see Ntlm in ntlm.py for more details
:param server_certificate_hash: The SHA256 hash string of the server certificate (DER encoded) NTLM is authenticating to. This is used to add
to the gss_channel_bindings_struct for Channel Binding Tokens support. If none is passed through then ntlm-auth
will not use Channel Binding Tokens when authenticating with the server which could cause issues if it is set to
only authenticate when these are present. This is only used for NTLMv2 authentication.
Message Attributes (Attributes not used to compute the message structure):
signature: An 8-byte character array that MUST contain the ASCII string 'NTLMSSP\0'
message_type: A 32-bit unsigned integer that indicates the message type. This field must be set to 0x00000003
negotiate_flags: A NEGOTIATE strucutre that contains a set of bit flags. These flags are the choices the client has made from the CHALLENGE_MESSAGE options
version: Contains the windows version info of the client. It is used only debugging purposes and are only set when NTLMSSP_NEGOTIATE_VERSION flag is set
mic: The message integrity for the NEGOTIATE_MESSAGE, CHALLENGE_MESSAGE and AUTHENTICATE_MESSAGE
lm_challenge_response: An LM_RESPONSE of LMv2_RESPONSE structure that contains the computed LM response to the challenge
nt_challenge_response: An NTLM_RESPONSE or NTLMv2_RESPONSE structure that contains the computed NT response to the challenge
domain_name: The domain or computer name hosting the user account, MUST be encoded in the negotiated character set
user_name: The name of the user to be authenticated, MUST be encoded in the negotiated character set
workstation: The name of the computer to which the user is logged on, MUST Be encoded in the negotiated character set
encrypted_random_session_key: The client's encrypted random session key
Non-Message Attributes (Attributes not used to compute the message structure):
exported_session_key: A randomly generated session key based on other keys, used to derive the SIGNKEY and SEALKEY
target_info: The AV_PAIR structure used in the nt response calculation
"""
def __init__(self, user_name, password, domain_name, workstation, challenge_message, ntlm_compatibility, server_certificate_hash):
self.signature = NTLM_SIGNATURE
self.message_type = struct.pack('<L', MessageTypes.NTLM_AUTHENTICATE)
self.negotiate_flags = challenge_message.negotiate_flags
self.version = get_version(self.negotiate_flags)
self.mic = None
if domain_name is None:
self.domain_name = ''
else:
self.domain_name = domain_name
if workstation is None:
self.workstation = ''
else:
self.workstation = workstation
if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_UNICODE:
self.negotiate_flags -= NegotiateFlags.NTLMSSP_NEGOTIATE_OEM
encoding_value = 'utf-16-le'
else:
encoding_value = 'ascii'
self.domain_name = self.domain_name.encode(encoding_value)
self.user_name = user_name.encode(encoding_value)
self.workstation = self.workstation.encode(encoding_value)
compute_response = ComputeResponse(user_name, password, domain_name, challenge_message,
ntlm_compatibility)
self.lm_challenge_response = compute_response.get_lm_challenge_response()
self.nt_challenge_response, key_exchange_key, target_info = compute_response.get_nt_challenge_response(
self.lm_challenge_response, server_certificate_hash)
self.target_info = target_info
if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_KEY_EXCH:
self.exported_session_key = get_random_export_session_key()
rc4_handle = ARC4(key_exchange_key)
self.encrypted_random_session_key = rc4_handle.update(self.exported_session_key)
else:
self.exported_session_key = key_exchange_key
self.encrypted_random_session_key = b''
self.negotiate_flags = struct.pack('<I', self.negotiate_flags)
def get_data(self):
if self.mic is None:
mic = b''
expected_body_length = self.EXPECTED_BODY_LENGTH
else:
mic = self.mic
expected_body_length = self.EXPECTED_BODY_LENGTH_WITH_MIC
payload_offset = expected_body_length
# DomainNameFields - 8 bytes
domain_name_len = struct.pack('<H', len(self.domain_name))
domain_name_max_len = struct.pack('<H', len(self.domain_name))
domain_name_buffer_offset = struct.pack('<I', payload_offset)
payload_offset += len(self.domain_name)
# UserNameFields - 8 bytes
user_name_len = struct.pack('<H', len(self.user_name))
user_name_max_len = struct.pack('<H', len(self.user_name))
user_name_buffer_offset = struct.pack('<I', payload_offset)
payload_offset += len(self.user_name)
# WorkstatonFields - 8 bytes
workstation_len = struct.pack('<H', len(self.workstation))
workstation_max_len = struct.pack('<H', len(self.workstation))
workstation_buffer_offset = struct.pack('<I', payload_offset)
payload_offset += len(self.workstation)
# LmChallengeResponseFields - 8 bytes
lm_challenge_response_len = struct.pack('<H', len(self.lm_challenge_response))
lm_challenge_response_max_len = struct.pack('<H', len(self.lm_challenge_response))
lm_challenge_response_buffer_offset = struct.pack('<I', payload_offset)
payload_offset += len(self.lm_challenge_response)
# NtChallengeResponseFields - 8 bytes
nt_challenge_response_len = struct.pack('<H', len(self.nt_challenge_response))
nt_challenge_response_max_len = struct.pack('<H', len(self.nt_challenge_response))
nt_challenge_response_buffer_offset = struct.pack('<I', payload_offset)
payload_offset += len(self.nt_challenge_response)
# EncryptedRandomSessionKeyFields - 8 bytes
encrypted_random_session_key_len = struct.pack('<H', len(self.encrypted_random_session_key))
encrypted_random_session_key_max_len = struct.pack('<H', len(self.encrypted_random_session_key))
encrypted_random_session_key_buffer_offset = struct.pack('<I', payload_offset)
payload_offset += len(self.encrypted_random_session_key)
# Payload - variable length
payload = self.domain_name
payload += self.user_name
payload += self.workstation
payload += self.lm_challenge_response
payload += self.nt_challenge_response
payload += self.encrypted_random_session_key
msg3 = self.signature
msg3 += self.message_type
msg3 += lm_challenge_response_len + lm_challenge_response_max_len + lm_challenge_response_buffer_offset
msg3 += nt_challenge_response_len + nt_challenge_response_max_len + nt_challenge_response_buffer_offset
msg3 += domain_name_len + domain_name_max_len + domain_name_buffer_offset
msg3 += user_name_len + user_name_max_len + user_name_buffer_offset
msg3 += workstation_len + workstation_max_len + workstation_buffer_offset
msg3 += encrypted_random_session_key_len + encrypted_random_session_key_max_len + encrypted_random_session_key_buffer_offset
msg3 += self.negotiate_flags
msg3 += self.version
msg3 += mic
# Adding the payload data to the message
msg3 += payload
return msg3
def add_mic(self, negotiate_message, challenge_message):
if self.target_info is not None:
av_flags = self.target_info[TargetInfo.MSV_AV_FLAGS]
if av_flags is not None and av_flags[1] == struct.pack("<L", AvFlags.MIC_PROVIDED):
self.mic = struct.pack("<IIII", 0, 0, 0, 0)
negotiate_data = negotiate_message.get_data()
challenge_data = challenge_message.get_data()
authenticate_data = self.get_data()
mic = hmac.new(self.exported_session_key,
(negotiate_data + challenge_data + authenticate_data)).digest()
self.mic = mic
def get_version(negotiate_flags):
# Check the negotiate_flag version is set, if it is make sure the version info is added to the data
if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_VERSION:
# TODO: Get the major and minor version of Windows instead of using default values
product_major_version = struct.pack('<B', 6)
product_minor_version = struct.pack('<B', 1)
product_build = struct.pack('<H', 7601)
version_reserved = b'\0' * 3
ntlm_revision_current = struct.pack('<B', 15)
version = product_major_version + product_minor_version + product_build + version_reserved + ntlm_revision_current
else:
version = b'\0' * 8
return version
def get_random_export_session_key():
return os.urandom(16)

View File

@ -1,146 +0,0 @@
# This library is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
import base64
import socket
import struct
from ntlm_auth.constants import NegotiateFlags
from ntlm_auth.messages import NegotiateMessage, ChallengeMessage, AuthenticateMessage
from ntlm_auth.session_security import SessionSecurity
"""
utility functions for Microsoft NTLM authentication
References:
[MS-NLMP]: NT LAN Manager (NTLM) Authentication Protocol Specification
http://download.microsoft.com/download/a/e/6/ae6e4142-aa58-45c6-8dcf-a657e5900cd3/%5BMS-NLMP%5D.pdf
[MS-NTHT]: NTLM Over HTTP Protocol Specification
http://download.microsoft.com/download/a/e/6/ae6e4142-aa58-45c6-8dcf-a657e5900cd3/%5BMS-NTHT%5D.pdf
Cntlm Authentication Proxy
http://cntlm.awk.cz/
NTLM Authorization Proxy Server
http://sourceforge.net/projects/ntlmaps/
Optimized Attack for NTLM2 Session Response
http://www.blackhat.com/presentations/bh-asia-04/bh-jp-04-pdfs/bh-jp-04-seki.pdf
"""
class Ntlm(object):
"""
Initialises the NTLM context to use when sending and receiving messages to and from the server. You should be
using this object as it supports NTLMv2 authenticate and it easier to use than before. It also brings in the
ability to use signing and sealing with session_security and generate a MIC structure.
:param ntlm_compatibility: The Lan Manager Compatibility Level to use withe the auth message - Default 3
This is set by an Administrator in the registry key
'HKLM\SYSTEM\CurrentControlSet\Control\Lsa\LmCompatibilityLevel'
The values correspond to the following;
0 : LM and NTLMv1
1 : LM, NTLMv1 and NTLMv1 with Extended Session Security
2 : NTLMv1 and NTLMv1 with Extended Session Security
3-5 : NTLMv2 Only
Note: Values 3 to 5 are no different as the client supports the same types
Attributes:
negotiate_flags: A NEGOTIATE structure that contains a set of bit flags. These flags are the options the client supports and are sent in the negotiate_message
ntlm_compatibility: The Lan Manager Compatibility Level, same as the input if supplied
negotiate_message: A NegotiateMessage object that is sent to the server
challenge_message: A ChallengeMessage object that has been created from the server response
authenticate_message: An AuthenticateMessage object that is sent to the server based on the ChallengeMessage
session_security: A SessionSecurity structure that can be used to sign and seal messages sent after the authentication challenge
"""
def __init__(self, ntlm_compatibility=3):
self.ntlm_compatibility = ntlm_compatibility
# Setting up our flags so the challenge message returns the target info block if supported
self.negotiate_flags = NegotiateFlags.NTLMSSP_NEGOTIATE_TARGET_INFO | \
NegotiateFlags.NTLMSSP_NEGOTIATE_128 | \
NegotiateFlags.NTLMSSP_NEGOTIATE_56 | \
NegotiateFlags.NTLMSSP_NEGOTIATE_UNICODE | \
NegotiateFlags.NTLMSSP_NEGOTIATE_VERSION | \
NegotiateFlags.NTLMSSP_NEGOTIATE_KEY_EXCH | \
NegotiateFlags.NTLMSSP_NEGOTIATE_ALWAYS_SIGN | \
NegotiateFlags.NTLMSSP_NEGOTIATE_SIGN | \
NegotiateFlags.NTLMSSP_NEGOTIATE_SEAL
# Setting the message types based on the ntlm_compatibility level
self._set_ntlm_compatibility_flags(self.ntlm_compatibility)
self.negotiate_message = None
self.challenge_message = None
self.authenticate_message = None
self.session_security = None
def create_negotiate_message(self, domain_name=None, workstation=None):
"""
Create an NTLM NEGOTIATE_MESSAGE
:param domain_name: The domain name of the user account we are authenticating with, default is None
:param worksation: The workstation we are using to authenticate with, default is None
:return: A base64 encoded string of the NEGOTIATE_MESSAGE
"""
self.negotiate_message = NegotiateMessage(self.negotiate_flags, domain_name, workstation)
return base64.b64encode(self.negotiate_message.get_data())
def parse_challenge_message(self, msg2):
"""
Parse the NTLM CHALLENGE_MESSAGE from the server and add it to the Ntlm context fields
:param msg2: A base64 encoded string of the CHALLENGE_MESSAGE
"""
msg2 = base64.b64decode(msg2)
self.challenge_message = ChallengeMessage(msg2)
def create_authenticate_message(self, user_name, password, domain_name=None, workstation=None, server_certificate_hash=None):
"""
Create an NTLM AUTHENTICATE_MESSAGE based on the Ntlm context and the previous messages sent and received
:param user_name: The user name of the user we are trying to authenticate with
:param password: The password of the user we are trying to authenticate with
:param domain_name: The domain name of the user account we are authenticated with, default is None
:param workstation: The workstation we are using to authenticate with, default is None
:param server_certificate_hash: The SHA256 hash string of the server certificate (DER encoded) NTLM is authenticating to. Used for Channel
Binding Tokens. If nothing is supplied then the CBT hash will not be sent. See messages.py AuthenticateMessage
for more details
:return: A base64 encoded string of the AUTHENTICATE_MESSAGE
"""
self.authenticate_message = AuthenticateMessage(user_name, password, domain_name, workstation,
self.challenge_message, self.ntlm_compatibility,
server_certificate_hash)
self.authenticate_message.add_mic(self.negotiate_message, self.challenge_message)
# Setups up the session_security context used to sign and seal messages if wanted
if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_SEAL or self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_SIGN:
self.session_security = SessionSecurity(struct.unpack("<I", self.authenticate_message.negotiate_flags)[0],
self.authenticate_message.exported_session_key)
return base64.b64encode(self.authenticate_message.get_data())
def _set_ntlm_compatibility_flags(self, ntlm_compatibility):
if (ntlm_compatibility >= 0) and (ntlm_compatibility <= 5):
if ntlm_compatibility == 0:
self.negotiate_flags |= NegotiateFlags.NTLMSSP_NEGOTIATE_NTLM | \
NegotiateFlags.NTLMSSP_NEGOTIATE_LM_KEY
elif ntlm_compatibility == 1:
self.negotiate_flags |= NegotiateFlags.NTLMSSP_NEGOTIATE_NTLM | \
NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY
else:
self.negotiate_flags |= NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY
else:
raise Exception("Unknown ntlm_compatibility level - expecting value between 0 and 5")

View File

@ -1,51 +0,0 @@
# This library is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
class ARC4(object):
state = None
i = 0
j = 0
def __init__(self, key):
# Split up the key into a list
if isinstance(key, str):
key = [ord(c) for c in key]
else:
key = [c for c in key]
#Key-scheduling algorithm (KSA)
self.state = [n for n in range(256)]
j = 0
for i in range(256):
j = (j + self.state[i] + key[i % len(key)]) % 256
self.state[i], self.state[j] = self.state[j], self.state[i]
def update(self, value):
chars = []
random_gen = self._random_generator()
for char in value:
if isinstance(value, str):
byte = ord(char)
else:
byte = char
updated_byte = byte ^ next(random_gen)
chars.append(updated_byte)
return bytes(bytearray(chars))
def _random_generator(self):
#Pseudo-Random Generation Algorithm (PRGA)
while True:
self.i = (self.i + 1) % 256
self.j = (self.j + self.state[self.i]) % 256
self.state[self.i], self.state[self.j] = self.state[self.j], self.state[self.i]
yield self.state[(self.state[self.i] + self.state[self.j]) % 256]

View File

@ -1,250 +0,0 @@
# This library is free software: you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/> or <http://www.gnu.org/licenses/lgpl.txt>.
import binascii
import hmac
import struct
import ntlm_auth.compute_keys as compkeys
from ntlm_auth.constants import NegotiateFlags, SignSealConstants
from ntlm_auth.rc4 import ARC4
class _NtlmMessageSignature1(object):
EXPECTED_BODY_LENGTH = 16
"""
[MS-NLMP] v28.0 2016-07-14
2.2.2.9.1 NTLMSSP_MESSAGE_SIGNATURE
This version of the NTLMSSP_MESSAGE_SIGNATURE structure MUST be used when the
NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY flag is not negotiated.
:param random_pad: A 4-byte array that contains the random pad for the emssage
:param checksum: A 4-byte array that contains the checksum for the message
:param seq_num: A 32-bit unsigned integer that contains the NTLM sequence number for this application message
"""
def __init__(self, random_pad, checksum, seq_num):
self.version = struct.pack("<I", 1)
self.random_pad = random_pad
self.checksum = checksum
self.seq_num = seq_num
def get_data(self):
signature = self.version
signature += self.random_pad
signature += self.checksum
signature += self.seq_num
assert self.EXPECTED_BODY_LENGTH == len(signature), "BODY_LENGTH: %d != signature: %d" % (
self.EXPECTED_BODY_LENGTH, len(signature))
return signature
class _NtlmMessageSignature2(object):
EXPECTED_BODY_LENGTH = 16
"""
[MS-NLMP] v28.0 2016-07-14
2.2.2.9.2 NTLMSSP_MESSAGE_SIGNATURE for Extended Session Security
This version of the NTLMSSP_MESSAGE_SIGNATURE structure MUST be used when the
NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY flag is negotiated
:param checksum: An 8-byte array that contains the checksum for the message
:param seq_num: A 32-bit unsigned integer that contains the NTLM sequence number for this application message
"""
def __init__(self, checksum, seq_num):
self.version = struct.pack("<I", 1)
self.checksum = checksum
self.seq_num = seq_num
def get_data(self):
signature = self.version
signature += self.checksum
signature += self.seq_num
assert self.EXPECTED_BODY_LENGTH == len(signature), "BODY_LENGTH: %d != signature: %d" % (
self.EXPECTED_BODY_LENGTH, len(signature))
return signature
class SessionSecurity(object):
"""
Initialises a security session context that can be used by libraries that call ntlm-auth to sign and seal
messages send to the server as well as verify and unseal messages that have been received from the server.
This is similar to the GSS_Wrap functions specified in the MS-NLMP document which does the same task.
:param negotiate_flags: The negotiate flag structure that has been negotiated with the server
:param exported_session_key: A 128-bit session key used to derive signing and sealing keys
:param source: The source of the message, only used in test scenarios when testing out a server sealing and unsealing
"""
def __init__(self, negotiate_flags, exported_session_key, source="client"):
self.negotiate_flags = negotiate_flags
self.outgoing_seq_num = 0
self.incoming_seq_num = 0
client_sealing_key = compkeys.get_seal_key(self.negotiate_flags, exported_session_key, SignSealConstants.CLIENT_SEALING)
server_sealing_key = compkeys.get_seal_key(self.negotiate_flags, exported_session_key, SignSealConstants.SERVER_SEALING)
if source == "client":
self.outgoing_signing_key = compkeys.get_sign_key(exported_session_key, SignSealConstants.CLIENT_SIGNING)
self.incoming_signing_key = compkeys.get_sign_key(exported_session_key, SignSealConstants.SERVER_SIGNING)
self.outgoing_handle = ARC4(client_sealing_key)
self.incoming_handle = ARC4(server_sealing_key)
elif source == "server":
self.outgoing_signing_key = compkeys.get_sign_key(exported_session_key, SignSealConstants.SERVER_SIGNING)
self.incoming_signing_key = compkeys.get_sign_key(exported_session_key, SignSealConstants.CLIENT_SIGNING)
self.outgoing_handle = ARC4(server_sealing_key)
self.incoming_handle = ARC4(client_sealing_key)
else:
raise Exception("Invalid source parameter %s, must be client or server" % source)
def wrap(self, message):
"""
[MS-NLMP] v28.0 2016-07-14
3.4.6 GSS_WrapEx()
Emulates the GSS_Wrap() implementation to sign and seal messages if the correct flags
are set.
@param message: The message data that will be wrapped
@return message: The message that has been sealed if flags are set
@return signature: The signature of the message, None if flags are not set
"""
if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_SEAL:
encrypted_message = self._seal_message(message)
signature = self._get_signature(message)
message = encrypted_message
elif self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_SIGN:
signature = self._get_signature(message)
else:
signature = None
return message, signature
def unwrap(self, message, signature):
"""
[MS-NLMP] v28.0 2016-07-14
3.4.7 GSS_UnwrapEx()
Emulates the GSS_Unwrap() implementation to unseal messages and verify the signature
sent matches what has been computed locally. Will throw an Exception if the signature
doesn't match
@param message: The message data received from the server
@param signature: The signature of the message
@return message: The message that has been unsealed if flags are set
"""
if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_SEAL:
message = self._unseal_message(message)
self._verify_signature(message, signature)
elif self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_SIGN:
self._verify_signature(message, signature)
return message
def _seal_message(self, message):
"""
[MS-NLMP] v28.0 2016-07-14
3.4.3 Message Confidentiality
Will generate an encrypted message using RC4 based on the ClientSealingKey
@param message: The message to be sealed (encrypted)
@return encrypted_message: The encrypted message
"""
encrypted_message = self.outgoing_handle.update(message)
return encrypted_message
def _unseal_message(self, message):
"""
[MS-NLMP] v28.0 2016-07-14
3.4.3 Message Confidentiality
Will generate a dencrypted message using RC4 based on the ServerSealingKey
@param message: The message to be unsealed (dencrypted)
@return decrypted_message: The decrypted message
"""
decrypted_message = self.incoming_handle.update(message)
return decrypted_message
def _get_signature(self, message):
"""
[MS-NLMP] v28.0 2016-07-14
3.4.4 Message Signature Functions
Will create the signature based on the message to send to the server. Depending on the negotiate_flags
set this could either be an NTLMv1 signature or NTLMv2 with Extended Session Security signature.
@param message: The message data that will be signed
@return signature: Either _NtlmMessageSignature1 or _NtlmMessageSignature2 depending on the flags set
"""
signature = calc_signature(message, self.negotiate_flags, self.outgoing_signing_key, self.outgoing_seq_num, self.outgoing_handle)
self.outgoing_seq_num += 1
return signature.get_data()
def _verify_signature(self, message, signature):
"""
Will verify that the signature received from the server matches up with the expected signature
computed locally. Will throw an exception if they do not match
@param message: The message data that is received from the server
@param signature: The signature of the message received from the server
"""
if self.negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY:
actual_checksum = signature[4:12]
actual_seq_num = struct.unpack("<I", signature[12:16])[0]
else:
actual_checksum = signature[8:12]
actual_seq_num = struct.unpack("<I", signature[12:16])[0]
expected_signature = calc_signature(message, self.negotiate_flags, self.incoming_signing_key, self.incoming_seq_num, self.incoming_handle)
expected_checksum = expected_signature.checksum
expected_seq_num = struct.unpack("<I", expected_signature.seq_num)[0]
if actual_checksum != expected_checksum:
raise Exception("The signature checksum does not match, message has been altered")
if actual_seq_num != expected_seq_num:
raise Exception("The signature sequence number does not match up, message not received in the correct sequence")
self.incoming_seq_num += 1
def calc_signature(message, negotiate_flags, signing_key, seq_num, handle):
seq_num = struct.pack("<I", seq_num)
if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_EXTENDED_SESSIONSECURITY:
checksum_hmac = hmac.new(signing_key, seq_num + message)
if negotiate_flags & NegotiateFlags.NTLMSSP_NEGOTIATE_KEY_EXCH:
checksum = handle.update(checksum_hmac.digest()[:8])
else:
checksum = checksum_hmac.digest()[:8]
signature = _NtlmMessageSignature2(checksum, seq_num)
else:
message_crc = binascii.crc32(message) % (1 << 32)
checksum = struct.pack("<I", message_crc)
random_pad = handle.update(struct.pack("<I", 0))
checksum = handle.update(checksum)
seq_num = handle.update(seq_num)
random_pad = struct.pack("<I", 0)
signature = _NtlmMessageSignature1(random_pad, checksum, seq_num)
return signature

View File

@ -1,68 +0,0 @@
"""
Original Author: Ian Clegg
Project: ntlmlib
URL: https://github.com/ianclegg/ntlmlib
License: Apache 2.0 License
Notes: Most of this code has been copied from the messages.py in the ntlmlib repo.
Some minor changes such as the name of the AV Pairs and extra comments have been added.
"""
import struct
try:
from collections import OrderedDict
except ImportError:
from ordereddict import OrderedDict
class TargetInfo(object):
MSV_AV_EOL = 0x00
MSV_AV_NB_COMPUTER_NAME = 0x01
MSV_AV_NB_DOMAIN_NAME = 0x02
MSV_AV_DNS_COMPUTER_NAME = 0x03
MSV_AV_DNS_DOMAIN_NAME = 0x04
MSV_AV_DNS_TREE_NAME = 0x05
MSV_AV_FLAGS = 0x06
MSV_AV_TIMESTAMP = 0x07
MSV_AV_SINGLE_HOST = 0x08
MSV_AV_TARGET_NAME = 0x09
MSV_AV_CHANNEL_BINDINGS = 0x0a
def __init__(self, data=None):
self.fields = OrderedDict()
if data is not None:
self.from_string(data)
def __setitem__(self, key, value):
self.fields[key] = (len(value), value)
def __getitem__(self, key):
if key in self.fields:
return self.fields[key]
return None
def __delitem__(self, key):
del self.fields[key]
def from_string(self, data):
attribute_type = 0xff
while attribute_type is not TargetInfo.MSV_AV_EOL:
# Parse the Attribute Value pair from the structure
attribute_type = struct.unpack('<H', data[:struct.calcsize('<H')])[0]
data = data[struct.calcsize('<H'):]
length = struct.unpack('<H', data[:struct.calcsize('<H')])[0]
data = data[struct.calcsize('<H'):]
# Add a new field to the object for the parse attribute value
self.fields[attribute_type] = (length, data[:length])
data = data[length:]
def get_data(self):
if TargetInfo.MSV_AV_EOL in self.fields:
del self.fields[TargetInfo.MSV_AV_EOL]
data = b''
for i in self.fields.keys():
data += struct.pack('<HH', i, self[i][0])
data += self[i][1]
# end with a NTLMSSP_AV_EOL
data += struct.pack('<HH', TargetInfo.MSV_AV_EOL, 0)
return data

View File

@ -1,127 +0,0 @@
# Copyright (c) 2009 Raymond Hettinger
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation files
# (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software,
# and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
from UserDict import DictMixin
class OrderedDict(dict, DictMixin):
def __init__(self, *args, **kwds):
if len(args) > 1:
raise TypeError('expected at most 1 arguments, got %d' % len(args))
try:
self.__end
except AttributeError:
self.clear()
self.update(*args, **kwds)
def clear(self):
self.__end = end = []
end += [None, end, end] # sentinel node for doubly linked list
self.__map = {} # key --> [key, prev, next]
dict.clear(self)
def __setitem__(self, key, value):
if key not in self:
end = self.__end
curr = end[1]
curr[2] = end[1] = self.__map[key] = [key, curr, end]
dict.__setitem__(self, key, value)
def __delitem__(self, key):
dict.__delitem__(self, key)
key, prev, next = self.__map.pop(key)
prev[2] = next
next[1] = prev
def __iter__(self):
end = self.__end
curr = end[2]
while curr is not end:
yield curr[0]
curr = curr[2]
def __reversed__(self):
end = self.__end
curr = end[1]
while curr is not end:
yield curr[0]
curr = curr[1]
def popitem(self, last=True):
if not self:
raise KeyError('dictionary is empty')
if last:
key = reversed(self).next()
else:
key = iter(self).next()
value = self.pop(key)
return key, value
def __reduce__(self):
items = [[k, self[k]] for k in self]
tmp = self.__map, self.__end
del self.__map, self.__end
inst_dict = vars(self).copy()
self.__map, self.__end = tmp
if inst_dict:
return (self.__class__, (items,), inst_dict)
return self.__class__, (items,)
def keys(self):
return list(self)
setdefault = DictMixin.setdefault
update = DictMixin.update
pop = DictMixin.pop
values = DictMixin.values
items = DictMixin.items
iterkeys = DictMixin.iterkeys
itervalues = DictMixin.itervalues
iteritems = DictMixin.iteritems
def __repr__(self):
if not self:
return '%s()' % (self.__class__.__name__,)
return '%s(%r)' % (self.__class__.__name__, self.items())
def copy(self):
return self.__class__(self)
@classmethod
def fromkeys(cls, iterable, value=None):
d = cls()
for key in iterable:
d[key] = value
return d
def __eq__(self, other):
if isinstance(other, OrderedDict):
if len(self) != len(other):
return False
for p, q in zip(self.items(), other.items()):
if p != q:
return False
return True
return dict.__eq__(self, other)
def __ne__(self, other):
return not self == other

View File

@ -1,90 +0,0 @@
# -*- coding: utf-8 -*-
"""
Pygments
~~~~~~~~
Pygments is a syntax highlighting package written in Python.
It is a generic syntax highlighter for general use in all kinds of software
such as forum systems, wikis or other applications that need to prettify
source code. Highlights are:
* a wide range of common languages and markup formats is supported
* special attention is paid to details, increasing quality by a fair amount
* support for new languages and formats are added easily
* a number of output formats, presently HTML, LaTeX, RTF, SVG, all image
formats that PIL supports, and ANSI sequences
* it is usable as a command-line tool and as a library
* ... and it highlights even Brainfuck!
The `Pygments tip`_ is installable with ``easy_install Pygments==dev``.
.. _Pygments tip:
http://bitbucket.org/birkenfeld/pygments-main/get/tip.zip#egg=Pygments-dev
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import sys
from pygments.util import StringIO, BytesIO
__version__ = '2.2.0'
__docformat__ = 'restructuredtext'
__all__ = ['lex', 'format', 'highlight']
def lex(code, lexer):
"""
Lex ``code`` with ``lexer`` and return an iterable of tokens.
"""
try:
return lexer.get_tokens(code)
except TypeError as err:
if (isinstance(err.args[0], str) and
('unbound method get_tokens' in err.args[0] or
'missing 1 required positional argument' in err.args[0])):
raise TypeError('lex() argument must be a lexer instance, '
'not a class')
raise
def format(tokens, formatter, outfile=None): # pylint: disable=redefined-builtin
"""
Format a tokenlist ``tokens`` with the formatter ``formatter``.
If ``outfile`` is given and a valid file object (an object
with a ``write`` method), the result will be written to it, otherwise
it is returned as a string.
"""
try:
if not outfile:
realoutfile = getattr(formatter, 'encoding', None) and BytesIO() or StringIO()
formatter.format(tokens, realoutfile)
return realoutfile.getvalue()
else:
formatter.format(tokens, outfile)
except TypeError as err:
if (isinstance(err.args[0], str) and
('unbound method format' in err.args[0] or
'missing 1 required positional argument' in err.args[0])):
raise TypeError('format() argument must be a formatter instance, '
'not a class')
raise
def highlight(code, lexer, formatter, outfile=None):
"""
Lex ``code`` with ``lexer`` and format it with the formatter ``formatter``.
If ``outfile`` is given and a valid file object (an object
with a ``write`` method), the result will be written to it, otherwise
it is returned as a string.
"""
return format(lex(code, lexer), formatter, outfile)
if __name__ == '__main__': # pragma: no cover
from pygments.cmdline import main
sys.exit(main(sys.argv))

View File

@ -1,573 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.cmdline
~~~~~~~~~~~~~~~~
Command line interface.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from __future__ import print_function
import os
import sys
import getopt
from textwrap import dedent
from pygments import __version__, highlight
from pygments.util import ClassNotFound, OptionError, docstring_headline, \
guess_decode, guess_decode_from_terminal, terminal_encoding
from pygments.lexers import get_all_lexers, get_lexer_by_name, guess_lexer, \
load_lexer_from_file, get_lexer_for_filename, find_lexer_class_for_filename
from pygments.lexers.special import TextLexer
from pygments.formatters.latex import LatexEmbeddedLexer, LatexFormatter
from pygments.formatters import get_all_formatters, get_formatter_by_name, \
load_formatter_from_file, get_formatter_for_filename, find_formatter_class
from pygments.formatters.terminal import TerminalFormatter
from pygments.formatters.terminal256 import Terminal256Formatter
from pygments.filters import get_all_filters, find_filter_class
from pygments.styles import get_all_styles, get_style_by_name
USAGE = """\
Usage: %s [-l <lexer> | -g] [-F <filter>[:<options>]] [-f <formatter>]
[-O <options>] [-P <option=value>] [-s] [-v] [-x] [-o <outfile>] [<infile>]
%s -S <style> -f <formatter> [-a <arg>] [-O <options>] [-P <option=value>]
%s -L [<which> ...]
%s -N <filename>
%s -H <type> <name>
%s -h | -V
Highlight the input file and write the result to <outfile>.
If no input file is given, use stdin, if -o is not given, use stdout.
If -s is passed, lexing will be done in "streaming" mode, reading and
highlighting one line at a time. This will only work properly with
lexers that have no constructs spanning multiple lines!
<lexer> is a lexer name (query all lexer names with -L). If -l is not
given, the lexer is guessed from the extension of the input file name
(this obviously doesn't work if the input is stdin). If -g is passed,
attempt to guess the lexer from the file contents, or pass through as
plain text if this fails (this can work for stdin).
Likewise, <formatter> is a formatter name, and will be guessed from
the extension of the output file name. If no output file is given,
the terminal formatter will be used by default.
The additional option -x allows custom lexers and formatters to be
loaded from a .py file relative to the current working directory. For
example, ``-l ./customlexer.py -x``. By default, this option expects a
file with a class named CustomLexer or CustomFormatter; you can also
specify your own class name with a colon (``-l ./lexer.py:MyLexer``).
Users should be very careful not to use this option with untrusted files,
because it will import and run them.
With the -O option, you can give the lexer and formatter a comma-
separated list of options, e.g. ``-O bg=light,python=cool``.
The -P option adds lexer and formatter options like the -O option, but
you can only give one option per -P. That way, the option value may
contain commas and equals signs, which it can't with -O, e.g.
``-P "heading=Pygments, the Python highlighter".
With the -F option, you can add filters to the token stream, you can
give options in the same way as for -O after a colon (note: there must
not be spaces around the colon).
The -O, -P and -F options can be given multiple times.
With the -S option, print out style definitions for style <style>
for formatter <formatter>. The argument given by -a is formatter
dependent.
The -L option lists lexers, formatters, styles or filters -- set
`which` to the thing you want to list (e.g. "styles"), or omit it to
list everything.
The -N option guesses and prints out a lexer name based solely on
the given filename. It does not take input or highlight anything.
If no specific lexer can be determined "text" is returned.
The -H option prints detailed help for the object <name> of type <type>,
where <type> is one of "lexer", "formatter" or "filter".
The -s option processes lines one at a time until EOF, rather than
waiting to process the entire file. This only works for stdin, and
is intended for streaming input such as you get from 'tail -f'.
Example usage: "tail -f sql.log | pygmentize -s -l sql"
The -v option prints a detailed traceback on unhandled exceptions,
which is useful for debugging and bug reports.
The -h option prints this help.
The -V option prints the package version.
"""
def _parse_options(o_strs):
opts = {}
if not o_strs:
return opts
for o_str in o_strs:
if not o_str.strip():
continue
o_args = o_str.split(',')
for o_arg in o_args:
o_arg = o_arg.strip()
try:
o_key, o_val = o_arg.split('=', 1)
o_key = o_key.strip()
o_val = o_val.strip()
except ValueError:
opts[o_arg] = True
else:
opts[o_key] = o_val
return opts
def _parse_filters(f_strs):
filters = []
if not f_strs:
return filters
for f_str in f_strs:
if ':' in f_str:
fname, fopts = f_str.split(':', 1)
filters.append((fname, _parse_options([fopts])))
else:
filters.append((f_str, {}))
return filters
def _print_help(what, name):
try:
if what == 'lexer':
cls = get_lexer_by_name(name)
print("Help on the %s lexer:" % cls.name)
print(dedent(cls.__doc__))
elif what == 'formatter':
cls = find_formatter_class(name)
print("Help on the %s formatter:" % cls.name)
print(dedent(cls.__doc__))
elif what == 'filter':
cls = find_filter_class(name)
print("Help on the %s filter:" % name)
print(dedent(cls.__doc__))
return 0
except (AttributeError, ValueError):
print("%s not found!" % what, file=sys.stderr)
return 1
def _print_list(what):
if what == 'lexer':
print()
print("Lexers:")
print("~~~~~~~")
info = []
for fullname, names, exts, _ in get_all_lexers():
tup = (', '.join(names)+':', fullname,
exts and '(filenames ' + ', '.join(exts) + ')' or '')
info.append(tup)
info.sort()
for i in info:
print(('* %s\n %s %s') % i)
elif what == 'formatter':
print()
print("Formatters:")
print("~~~~~~~~~~~")
info = []
for cls in get_all_formatters():
doc = docstring_headline(cls)
tup = (', '.join(cls.aliases) + ':', doc, cls.filenames and
'(filenames ' + ', '.join(cls.filenames) + ')' or '')
info.append(tup)
info.sort()
for i in info:
print(('* %s\n %s %s') % i)
elif what == 'filter':
print()
print("Filters:")
print("~~~~~~~~")
for name in get_all_filters():
cls = find_filter_class(name)
print("* " + name + ':')
print(" %s" % docstring_headline(cls))
elif what == 'style':
print()
print("Styles:")
print("~~~~~~~")
for name in get_all_styles():
cls = get_style_by_name(name)
print("* " + name + ':')
print(" %s" % docstring_headline(cls))
def main_inner(popts, args, usage):
opts = {}
O_opts = []
P_opts = []
F_opts = []
for opt, arg in popts:
if opt == '-O':
O_opts.append(arg)
elif opt == '-P':
P_opts.append(arg)
elif opt == '-F':
F_opts.append(arg)
opts[opt] = arg
if opts.pop('-h', None) is not None:
print(usage)
return 0
if opts.pop('-V', None) is not None:
print('Pygments version %s, (c) 2006-2017 by Georg Brandl.' % __version__)
return 0
# handle ``pygmentize -L``
L_opt = opts.pop('-L', None)
if L_opt is not None:
if opts:
print(usage, file=sys.stderr)
return 2
# print version
main(['', '-V'])
if not args:
args = ['lexer', 'formatter', 'filter', 'style']
for arg in args:
_print_list(arg.rstrip('s'))
return 0
# handle ``pygmentize -H``
H_opt = opts.pop('-H', None)
if H_opt is not None:
if opts or len(args) != 2:
print(usage, file=sys.stderr)
return 2
what, name = args # pylint: disable=unbalanced-tuple-unpacking
if what not in ('lexer', 'formatter', 'filter'):
print(usage, file=sys.stderr)
return 2
return _print_help(what, name)
# parse -O options
parsed_opts = _parse_options(O_opts)
opts.pop('-O', None)
# parse -P options
for p_opt in P_opts:
try:
name, value = p_opt.split('=', 1)
except ValueError:
parsed_opts[p_opt] = True
else:
parsed_opts[name] = value
opts.pop('-P', None)
# encodings
inencoding = parsed_opts.get('inencoding', parsed_opts.get('encoding'))
outencoding = parsed_opts.get('outencoding', parsed_opts.get('encoding'))
# handle ``pygmentize -N``
infn = opts.pop('-N', None)
if infn is not None:
lexer = find_lexer_class_for_filename(infn)
if lexer is None:
lexer = TextLexer
print(lexer.aliases[0])
return 0
# handle ``pygmentize -S``
S_opt = opts.pop('-S', None)
a_opt = opts.pop('-a', None)
if S_opt is not None:
f_opt = opts.pop('-f', None)
if not f_opt:
print(usage, file=sys.stderr)
return 2
if opts or args:
print(usage, file=sys.stderr)
return 2
try:
parsed_opts['style'] = S_opt
fmter = get_formatter_by_name(f_opt, **parsed_opts)
except ClassNotFound as err:
print(err, file=sys.stderr)
return 1
print(fmter.get_style_defs(a_opt or ''))
return 0
# if no -S is given, -a is not allowed
if a_opt is not None:
print(usage, file=sys.stderr)
return 2
# parse -F options
F_opts = _parse_filters(F_opts)
opts.pop('-F', None)
allow_custom_lexer_formatter = False
# -x: allow custom (eXternal) lexers and formatters
if opts.pop('-x', None) is not None:
allow_custom_lexer_formatter = True
# select lexer
lexer = None
# given by name?
lexername = opts.pop('-l', None)
if lexername:
# custom lexer, located relative to user's cwd
if allow_custom_lexer_formatter and '.py' in lexername:
try:
if ':' in lexername:
filename, name = lexername.rsplit(':', 1)
lexer = load_lexer_from_file(filename, name,
**parsed_opts)
else:
lexer = load_lexer_from_file(lexername, **parsed_opts)
except ClassNotFound as err:
print('Error:', err, file=sys.stderr)
return 1
else:
try:
lexer = get_lexer_by_name(lexername, **parsed_opts)
except (OptionError, ClassNotFound) as err:
print('Error:', err, file=sys.stderr)
return 1
# read input code
code = None
if args:
if len(args) > 1:
print(usage, file=sys.stderr)
return 2
if '-s' in opts:
print('Error: -s option not usable when input file specified',
file=sys.stderr)
return 2
infn = args[0]
try:
with open(infn, 'rb') as infp:
code = infp.read()
except Exception as err:
print('Error: cannot read infile:', err, file=sys.stderr)
return 1
if not inencoding:
code, inencoding = guess_decode(code)
# do we have to guess the lexer?
if not lexer:
try:
lexer = get_lexer_for_filename(infn, code, **parsed_opts)
except ClassNotFound as err:
if '-g' in opts:
try:
lexer = guess_lexer(code, **parsed_opts)
except ClassNotFound:
lexer = TextLexer(**parsed_opts)
else:
print('Error:', err, file=sys.stderr)
return 1
except OptionError as err:
print('Error:', err, file=sys.stderr)
return 1
elif '-s' not in opts: # treat stdin as full file (-s support is later)
# read code from terminal, always in binary mode since we want to
# decode ourselves and be tolerant with it
if sys.version_info > (3,):
# Python 3: we have to use .buffer to get a binary stream
code = sys.stdin.buffer.read()
else:
code = sys.stdin.read()
if not inencoding:
code, inencoding = guess_decode_from_terminal(code, sys.stdin)
# else the lexer will do the decoding
if not lexer:
try:
lexer = guess_lexer(code, **parsed_opts)
except ClassNotFound:
lexer = TextLexer(**parsed_opts)
else: # -s option needs a lexer with -l
if not lexer:
print('Error: when using -s a lexer has to be selected with -l',
file=sys.stderr)
return 2
# process filters
for fname, fopts in F_opts:
try:
lexer.add_filter(fname, **fopts)
except ClassNotFound as err:
print('Error:', err, file=sys.stderr)
return 1
# select formatter
outfn = opts.pop('-o', None)
fmter = opts.pop('-f', None)
if fmter:
# custom formatter, located relative to user's cwd
if allow_custom_lexer_formatter and '.py' in fmter:
try:
if ':' in fmter:
file, fmtername = fmter.rsplit(':', 1)
fmter = load_formatter_from_file(file, fmtername,
**parsed_opts)
else:
fmter = load_formatter_from_file(fmter, **parsed_opts)
except ClassNotFound as err:
print('Error:', err, file=sys.stderr)
return 1
else:
try:
fmter = get_formatter_by_name(fmter, **parsed_opts)
except (OptionError, ClassNotFound) as err:
print('Error:', err, file=sys.stderr)
return 1
if outfn:
if not fmter:
try:
fmter = get_formatter_for_filename(outfn, **parsed_opts)
except (OptionError, ClassNotFound) as err:
print('Error:', err, file=sys.stderr)
return 1
try:
outfile = open(outfn, 'wb')
except Exception as err:
print('Error: cannot open outfile:', err, file=sys.stderr)
return 1
else:
if not fmter:
if '256' in os.environ.get('TERM', ''):
fmter = Terminal256Formatter(**parsed_opts)
else:
fmter = TerminalFormatter(**parsed_opts)
if sys.version_info > (3,):
# Python 3: we have to use .buffer to get a binary stream
outfile = sys.stdout.buffer
else:
outfile = sys.stdout
# determine output encoding if not explicitly selected
if not outencoding:
if outfn:
# output file? use lexer encoding for now (can still be None)
fmter.encoding = inencoding
else:
# else use terminal encoding
fmter.encoding = terminal_encoding(sys.stdout)
# provide coloring under Windows, if possible
if not outfn and sys.platform in ('win32', 'cygwin') and \
fmter.name in ('Terminal', 'Terminal256'): # pragma: no cover
# unfortunately colorama doesn't support binary streams on Py3
if sys.version_info > (3,):
from pygments.util import UnclosingTextIOWrapper
outfile = UnclosingTextIOWrapper(outfile, encoding=fmter.encoding)
fmter.encoding = None
try:
import colorama.initialise
except ImportError:
pass
else:
outfile = colorama.initialise.wrap_stream(
outfile, convert=None, strip=None, autoreset=False, wrap=True)
# When using the LaTeX formatter and the option `escapeinside` is
# specified, we need a special lexer which collects escaped text
# before running the chosen language lexer.
escapeinside = parsed_opts.get('escapeinside', '')
if len(escapeinside) == 2 and isinstance(fmter, LatexFormatter):
left = escapeinside[0]
right = escapeinside[1]
lexer = LatexEmbeddedLexer(left, right, lexer)
# ... and do it!
if '-s' not in opts:
# process whole input as per normal...
highlight(code, lexer, fmter, outfile)
return 0
else:
# line by line processing of stdin (eg: for 'tail -f')...
try:
while 1:
if sys.version_info > (3,):
# Python 3: we have to use .buffer to get a binary stream
line = sys.stdin.buffer.readline()
else:
line = sys.stdin.readline()
if not line:
break
if not inencoding:
line = guess_decode_from_terminal(line, sys.stdin)[0]
highlight(line, lexer, fmter, outfile)
if hasattr(outfile, 'flush'):
outfile.flush()
return 0
except KeyboardInterrupt: # pragma: no cover
return 0
def main(args=sys.argv):
"""
Main command line entry point.
"""
usage = USAGE % ((args[0],) * 6)
try:
popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:vhVHgsx")
except getopt.GetoptError:
print(usage, file=sys.stderr)
return 2
try:
return main_inner(popts, args, usage)
except Exception:
if '-v' in dict(popts):
print(file=sys.stderr)
print('*' * 65, file=sys.stderr)
print('An unhandled exception occurred while highlighting.',
file=sys.stderr)
print('Please report the whole traceback to the issue tracker at',
file=sys.stderr)
print('<https://bitbucket.org/birkenfeld/pygments-main/issues>.',
file=sys.stderr)
print('*' * 65, file=sys.stderr)
print(file=sys.stderr)
raise
import traceback
info = traceback.format_exception(*sys.exc_info())
msg = info[-1].strip()
if len(info) >= 3:
# extract relevant file and position info
msg += '\n (f%s)' % info[-2].split('\n')[0].strip()[1:]
print(file=sys.stderr)
print('*** Error while highlighting:', file=sys.stderr)
print(msg, file=sys.stderr)
print('*** If this is a bug you want to report, please rerun with -v.',
file=sys.stderr)
return 1

View File

@ -1,74 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.console
~~~~~~~~~~~~~~~~
Format colored console output.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
esc = "\x1b["
codes = {}
codes[""] = ""
codes["reset"] = esc + "39;49;00m"
codes["bold"] = esc + "01m"
codes["faint"] = esc + "02m"
codes["standout"] = esc + "03m"
codes["underline"] = esc + "04m"
codes["blink"] = esc + "05m"
codes["overline"] = esc + "06m"
dark_colors = ["black", "darkred", "darkgreen", "brown", "darkblue",
"purple", "teal", "lightgray"]
light_colors = ["darkgray", "red", "green", "yellow", "blue",
"fuchsia", "turquoise", "white"]
x = 30
for d, l in zip(dark_colors, light_colors):
codes[d] = esc + "%im" % x
codes[l] = esc + "%i;01m" % x
x += 1
del d, l, x
codes["darkteal"] = codes["turquoise"]
codes["darkyellow"] = codes["brown"]
codes["fuscia"] = codes["fuchsia"]
codes["white"] = codes["bold"]
def reset_color():
return codes["reset"]
def colorize(color_key, text):
return codes[color_key] + text + codes["reset"]
def ansiformat(attr, text):
"""
Format ``text`` with a color and/or some attributes::
color normal color
*color* bold color
_color_ underlined color
+color+ blinking color
"""
result = []
if attr[:1] == attr[-1:] == '+':
result.append(codes['blink'])
attr = attr[1:-1]
if attr[:1] == attr[-1:] == '*':
result.append(codes['bold'])
attr = attr[1:-1]
if attr[:1] == attr[-1:] == '_':
result.append(codes['underline'])
attr = attr[1:-1]
result.append(codes[attr])
result.append(text)
result.append(codes['reset'])
return ''.join(result)

View File

@ -1,74 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.filter
~~~~~~~~~~~~~~~
Module that implements the default filter.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
def apply_filters(stream, filters, lexer=None):
"""
Use this method to apply an iterable of filters to
a stream. If lexer is given it's forwarded to the
filter, otherwise the filter receives `None`.
"""
def _apply(filter_, stream):
for token in filter_.filter(lexer, stream):
yield token
for filter_ in filters:
stream = _apply(filter_, stream)
return stream
def simplefilter(f):
"""
Decorator that converts a function into a filter::
@simplefilter
def lowercase(self, lexer, stream, options):
for ttype, value in stream:
yield ttype, value.lower()
"""
return type(f.__name__, (FunctionFilter,), {
'__module__': getattr(f, '__module__'),
'__doc__': f.__doc__,
'function': f,
})
class Filter(object):
"""
Default filter. Subclass this class or use the `simplefilter`
decorator to create own filters.
"""
def __init__(self, **options):
self.options = options
def filter(self, lexer, stream):
raise NotImplementedError()
class FunctionFilter(Filter):
"""
Abstract class used by `simplefilter` to create simple
function filters on the fly. The `simplefilter` decorator
automatically creates subclasses of this class for
functions passed to it.
"""
function = None
def __init__(self, **options):
if not hasattr(self, 'function'):
raise TypeError('%r used without bound function' %
self.__class__.__name__)
Filter.__init__(self, **options)
def filter(self, lexer, stream):
# pylint: disable=not-callable
for ttype, value in self.function(lexer, stream, self.options):
yield ttype, value

View File

@ -1,350 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.filters
~~~~~~~~~~~~~~~~
Module containing filter lookup functions and default
filters.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import re
from pygments.token import String, Comment, Keyword, Name, Error, Whitespace, \
string_to_tokentype
from pygments.filter import Filter
from pygments.util import get_list_opt, get_int_opt, get_bool_opt, \
get_choice_opt, ClassNotFound, OptionError, text_type, string_types
from pygments.plugin import find_plugin_filters
def find_filter_class(filtername):
"""Lookup a filter by name. Return None if not found."""
if filtername in FILTERS:
return FILTERS[filtername]
for name, cls in find_plugin_filters():
if name == filtername:
return cls
return None
def get_filter_by_name(filtername, **options):
"""Return an instantiated filter.
Options are passed to the filter initializer if wanted.
Raise a ClassNotFound if not found.
"""
cls = find_filter_class(filtername)
if cls:
return cls(**options)
else:
raise ClassNotFound('filter %r not found' % filtername)
def get_all_filters():
"""Return a generator of all filter names."""
for name in FILTERS:
yield name
for name, _ in find_plugin_filters():
yield name
def _replace_special(ttype, value, regex, specialttype,
replacefunc=lambda x: x):
last = 0
for match in regex.finditer(value):
start, end = match.start(), match.end()
if start != last:
yield ttype, value[last:start]
yield specialttype, replacefunc(value[start:end])
last = end
if last != len(value):
yield ttype, value[last:]
class CodeTagFilter(Filter):
"""Highlight special code tags in comments and docstrings.
Options accepted:
`codetags` : list of strings
A list of strings that are flagged as code tags. The default is to
highlight ``XXX``, ``TODO``, ``BUG`` and ``NOTE``.
"""
def __init__(self, **options):
Filter.__init__(self, **options)
tags = get_list_opt(options, 'codetags',
['XXX', 'TODO', 'BUG', 'NOTE'])
self.tag_re = re.compile(r'\b(%s)\b' % '|'.join([
re.escape(tag) for tag in tags if tag
]))
def filter(self, lexer, stream):
regex = self.tag_re
for ttype, value in stream:
if ttype in String.Doc or \
ttype in Comment and \
ttype not in Comment.Preproc:
for sttype, svalue in _replace_special(ttype, value, regex,
Comment.Special):
yield sttype, svalue
else:
yield ttype, value
class KeywordCaseFilter(Filter):
"""Convert keywords to lowercase or uppercase or capitalize them, which
means first letter uppercase, rest lowercase.
This can be useful e.g. if you highlight Pascal code and want to adapt the
code to your styleguide.
Options accepted:
`case` : string
The casing to convert keywords to. Must be one of ``'lower'``,
``'upper'`` or ``'capitalize'``. The default is ``'lower'``.
"""
def __init__(self, **options):
Filter.__init__(self, **options)
case = get_choice_opt(options, 'case',
['lower', 'upper', 'capitalize'], 'lower')
self.convert = getattr(text_type, case)
def filter(self, lexer, stream):
for ttype, value in stream:
if ttype in Keyword:
yield ttype, self.convert(value)
else:
yield ttype, value
class NameHighlightFilter(Filter):
"""Highlight a normal Name (and Name.*) token with a different token type.
Example::
filter = NameHighlightFilter(
names=['foo', 'bar', 'baz'],
tokentype=Name.Function,
)
This would highlight the names "foo", "bar" and "baz"
as functions. `Name.Function` is the default token type.
Options accepted:
`names` : list of strings
A list of names that should be given the different token type.
There is no default.
`tokentype` : TokenType or string
A token type or a string containing a token type name that is
used for highlighting the strings in `names`. The default is
`Name.Function`.
"""
def __init__(self, **options):
Filter.__init__(self, **options)
self.names = set(get_list_opt(options, 'names', []))
tokentype = options.get('tokentype')
if tokentype:
self.tokentype = string_to_tokentype(tokentype)
else:
self.tokentype = Name.Function
def filter(self, lexer, stream):
for ttype, value in stream:
if ttype in Name and value in self.names:
yield self.tokentype, value
else:
yield ttype, value
class ErrorToken(Exception):
pass
class RaiseOnErrorTokenFilter(Filter):
"""Raise an exception when the lexer generates an error token.
Options accepted:
`excclass` : Exception class
The exception class to raise.
The default is `pygments.filters.ErrorToken`.
.. versionadded:: 0.8
"""
def __init__(self, **options):
Filter.__init__(self, **options)
self.exception = options.get('excclass', ErrorToken)
try:
# issubclass() will raise TypeError if first argument is not a class
if not issubclass(self.exception, Exception):
raise TypeError
except TypeError:
raise OptionError('excclass option is not an exception class')
def filter(self, lexer, stream):
for ttype, value in stream:
if ttype is Error:
raise self.exception(value)
yield ttype, value
class VisibleWhitespaceFilter(Filter):
"""Convert tabs, newlines and/or spaces to visible characters.
Options accepted:
`spaces` : string or bool
If this is a one-character string, spaces will be replaces by this string.
If it is another true value, spaces will be replaced by ``·`` (unicode
MIDDLE DOT). If it is a false value, spaces will not be replaced. The
default is ``False``.
`tabs` : string or bool
The same as for `spaces`, but the default replacement character is ``»``
(unicode RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK). The default value
is ``False``. Note: this will not work if the `tabsize` option for the
lexer is nonzero, as tabs will already have been expanded then.
`tabsize` : int
If tabs are to be replaced by this filter (see the `tabs` option), this
is the total number of characters that a tab should be expanded to.
The default is ``8``.
`newlines` : string or bool
The same as for `spaces`, but the default replacement character is ``¶``
(unicode PILCROW SIGN). The default value is ``False``.
`wstokentype` : bool
If true, give whitespace the special `Whitespace` token type. This allows
styling the visible whitespace differently (e.g. greyed out), but it can
disrupt background colors. The default is ``True``.
.. versionadded:: 0.8
"""
def __init__(self, **options):
Filter.__init__(self, **options)
for name, default in [('spaces', u'·'),
('tabs', u'»'),
('newlines', u'')]:
opt = options.get(name, False)
if isinstance(opt, string_types) and len(opt) == 1:
setattr(self, name, opt)
else:
setattr(self, name, (opt and default or ''))
tabsize = get_int_opt(options, 'tabsize', 8)
if self.tabs:
self.tabs += ' ' * (tabsize - 1)
if self.newlines:
self.newlines += '\n'
self.wstt = get_bool_opt(options, 'wstokentype', True)
def filter(self, lexer, stream):
if self.wstt:
spaces = self.spaces or u' '
tabs = self.tabs or u'\t'
newlines = self.newlines or u'\n'
regex = re.compile(r'\s')
def replacefunc(wschar):
if wschar == ' ':
return spaces
elif wschar == '\t':
return tabs
elif wschar == '\n':
return newlines
return wschar
for ttype, value in stream:
for sttype, svalue in _replace_special(ttype, value, regex,
Whitespace, replacefunc):
yield sttype, svalue
else:
spaces, tabs, newlines = self.spaces, self.tabs, self.newlines
# simpler processing
for ttype, value in stream:
if spaces:
value = value.replace(' ', spaces)
if tabs:
value = value.replace('\t', tabs)
if newlines:
value = value.replace('\n', newlines)
yield ttype, value
class GobbleFilter(Filter):
"""Gobbles source code lines (eats initial characters).
This filter drops the first ``n`` characters off every line of code. This
may be useful when the source code fed to the lexer is indented by a fixed
amount of space that isn't desired in the output.
Options accepted:
`n` : int
The number of characters to gobble.
.. versionadded:: 1.2
"""
def __init__(self, **options):
Filter.__init__(self, **options)
self.n = get_int_opt(options, 'n', 0)
def gobble(self, value, left):
if left < len(value):
return value[left:], 0
else:
return u'', left - len(value)
def filter(self, lexer, stream):
n = self.n
left = n # How many characters left to gobble.
for ttype, value in stream:
# Remove ``left`` tokens from first line, ``n`` from all others.
parts = value.split('\n')
(parts[0], left) = self.gobble(parts[0], left)
for i in range(1, len(parts)):
(parts[i], left) = self.gobble(parts[i], n)
value = u'\n'.join(parts)
if value != '':
yield ttype, value
class TokenMergeFilter(Filter):
"""Merges consecutive tokens with the same token type in the output
stream of a lexer.
.. versionadded:: 1.2
"""
def __init__(self, **options):
Filter.__init__(self, **options)
def filter(self, lexer, stream):
current_type = None
current_value = None
for ttype, value in stream:
if ttype is current_type:
current_value += value
else:
if current_type is not None:
yield current_type, current_value
current_type = ttype
current_value = value
if current_type is not None:
yield current_type, current_value
FILTERS = {
'codetagify': CodeTagFilter,
'keywordcase': KeywordCaseFilter,
'highlight': NameHighlightFilter,
'raiseonerror': RaiseOnErrorTokenFilter,
'whitespace': VisibleWhitespaceFilter,
'gobble': GobbleFilter,
'tokenmerge': TokenMergeFilter,
}

View File

@ -1,95 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.formatter
~~~~~~~~~~~~~~~~~~
Base formatter class.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import codecs
from pygments.util import get_bool_opt, string_types
from pygments.styles import get_style_by_name
__all__ = ['Formatter']
def _lookup_style(style):
if isinstance(style, string_types):
return get_style_by_name(style)
return style
class Formatter(object):
"""
Converts a token stream to text.
Options accepted:
``style``
The style to use, can be a string or a Style subclass
(default: "default"). Not used by e.g. the
TerminalFormatter.
``full``
Tells the formatter to output a "full" document, i.e.
a complete self-contained document. This doesn't have
any effect for some formatters (default: false).
``title``
If ``full`` is true, the title that should be used to
caption the document (default: '').
``encoding``
If given, must be an encoding name. This will be used to
convert the Unicode token strings to byte strings in the
output. If it is "" or None, Unicode strings will be written
to the output file, which most file-like objects do not
support (default: None).
``outencoding``
Overrides ``encoding`` if given.
"""
#: Name of the formatter
name = None
#: Shortcuts for the formatter
aliases = []
#: fn match rules
filenames = []
#: If True, this formatter outputs Unicode strings when no encoding
#: option is given.
unicodeoutput = True
def __init__(self, **options):
self.style = _lookup_style(options.get('style', 'default'))
self.full = get_bool_opt(options, 'full', False)
self.title = options.get('title', '')
self.encoding = options.get('encoding', None) or None
if self.encoding in ('guess', 'chardet'):
# can happen for e.g. pygmentize -O encoding=guess
self.encoding = 'utf-8'
self.encoding = options.get('outencoding') or self.encoding
self.options = options
def get_style_defs(self, arg=''):
"""
Return the style definitions for the current style as a string.
``arg`` is an additional argument whose meaning depends on the
formatter used. Note that ``arg`` can also be a list or tuple
for some formatters like the html formatter.
"""
return ''
def format(self, tokensource, outfile):
"""
Format ``tokensource``, an iterable of ``(tokentype, tokenstring)``
tuples and write it into ``outfile``.
"""
if self.encoding:
# wrap the outfile in a StreamWriter
outfile = codecs.lookup(self.encoding)[3](outfile)
return self.format_unencoded(tokensource, outfile)

View File

@ -1,153 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.formatters
~~~~~~~~~~~~~~~~~~~
Pygments formatters.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import re
import sys
import types
import fnmatch
from os.path import basename
from pygments.formatters._mapping import FORMATTERS
from pygments.plugin import find_plugin_formatters
from pygments.util import ClassNotFound, itervalues
__all__ = ['get_formatter_by_name', 'get_formatter_for_filename',
'get_all_formatters', 'load_formatter_from_file'] + list(FORMATTERS)
_formatter_cache = {} # classes by name
_pattern_cache = {}
def _fn_matches(fn, glob):
"""Return whether the supplied file name fn matches pattern filename."""
if glob not in _pattern_cache:
pattern = _pattern_cache[glob] = re.compile(fnmatch.translate(glob))
return pattern.match(fn)
return _pattern_cache[glob].match(fn)
def _load_formatters(module_name):
"""Load a formatter (and all others in the module too)."""
mod = __import__(module_name, None, None, ['__all__'])
for formatter_name in mod.__all__:
cls = getattr(mod, formatter_name)
_formatter_cache[cls.name] = cls
def get_all_formatters():
"""Return a generator for all formatter classes."""
# NB: this returns formatter classes, not info like get_all_lexers().
for info in itervalues(FORMATTERS):
if info[1] not in _formatter_cache:
_load_formatters(info[0])
yield _formatter_cache[info[1]]
for _, formatter in find_plugin_formatters():
yield formatter
def find_formatter_class(alias):
"""Lookup a formatter by alias.
Returns None if not found.
"""
for module_name, name, aliases, _, _ in itervalues(FORMATTERS):
if alias in aliases:
if name not in _formatter_cache:
_load_formatters(module_name)
return _formatter_cache[name]
for _, cls in find_plugin_formatters():
if alias in cls.aliases:
return cls
def get_formatter_by_name(_alias, **options):
"""Lookup and instantiate a formatter by alias.
Raises ClassNotFound if not found.
"""
cls = find_formatter_class(_alias)
if cls is None:
raise ClassNotFound("no formatter found for name %r" % _alias)
return cls(**options)
def load_formatter_from_file(filename, formattername="CustomFormatter",
**options):
"""Load a formatter from a file.
This method expects a file located relative to the current working
directory, which contains a class named CustomFormatter. By default,
it expects the Formatter to be named CustomFormatter; you can specify
your own class name as the second argument to this function.
Users should be very careful with the input, because this method
is equivalent to running eval on the input file.
Raises ClassNotFound if there are any problems importing the Formatter.
.. versionadded:: 2.2
"""
try:
# This empty dict will contain the namespace for the exec'd file
custom_namespace = {}
exec(open(filename, 'rb').read(), custom_namespace)
# Retrieve the class `formattername` from that namespace
if formattername not in custom_namespace:
raise ClassNotFound('no valid %s class found in %s' %
(formattername, filename))
formatter_class = custom_namespace[formattername]
# And finally instantiate it with the options
return formatter_class(**options)
except IOError as err:
raise ClassNotFound('cannot read %s' % filename)
except ClassNotFound as err:
raise
except Exception as err:
raise ClassNotFound('error when loading custom formatter: %s' % err)
def get_formatter_for_filename(fn, **options):
"""Lookup and instantiate a formatter by filename pattern.
Raises ClassNotFound if not found.
"""
fn = basename(fn)
for modname, name, _, filenames, _ in itervalues(FORMATTERS):
for filename in filenames:
if _fn_matches(fn, filename):
if name not in _formatter_cache:
_load_formatters(modname)
return _formatter_cache[name](**options)
for cls in find_plugin_formatters():
for filename in cls.filenames:
if _fn_matches(fn, filename):
return cls(**options)
raise ClassNotFound("no formatter found for file name %r" % fn)
class _automodule(types.ModuleType):
"""Automatically import formatters."""
def __getattr__(self, name):
info = FORMATTERS.get(name)
if info:
_load_formatters(info[0])
cls = _formatter_cache[info[1]]
setattr(self, name, cls)
return cls
raise AttributeError(name)
oldmod = sys.modules[__name__]
newmod = _automodule(__name__)
newmod.__dict__.update(oldmod.__dict__)
sys.modules[__name__] = newmod
del newmod.newmod, newmod.oldmod, newmod.sys, newmod.types

View File

@ -1,85 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.formatters._mapping
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Formatter mapping definitions. This file is generated by itself. Everytime
you change something on a builtin formatter definition, run this script from
the formatters folder to update it.
Do not alter the FORMATTERS dictionary by hand.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from __future__ import print_function
FORMATTERS = {
'BBCodeFormatter': ('pygments.formatters.bbcode', 'BBCode', ('bbcode', 'bb'), (), 'Format tokens with BBcodes. These formatting codes are used by many bulletin boards, so you can highlight your sourcecode with pygments before posting it there.'),
'BmpImageFormatter': ('pygments.formatters.img', 'img_bmp', ('bmp', 'bitmap'), ('*.bmp',), 'Create a bitmap image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
'GifImageFormatter': ('pygments.formatters.img', 'img_gif', ('gif',), ('*.gif',), 'Create a GIF image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
'HtmlFormatter': ('pygments.formatters.html', 'HTML', ('html',), ('*.html', '*.htm'), "Format tokens as HTML 4 ``<span>`` tags within a ``<pre>`` tag, wrapped in a ``<div>`` tag. The ``<div>``'s CSS class can be set by the `cssclass` option."),
'IRCFormatter': ('pygments.formatters.irc', 'IRC', ('irc', 'IRC'), (), 'Format tokens with IRC color sequences'),
'ImageFormatter': ('pygments.formatters.img', 'img', ('img', 'IMG', 'png'), ('*.png',), 'Create a PNG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
'JpgImageFormatter': ('pygments.formatters.img', 'img_jpg', ('jpg', 'jpeg'), ('*.jpg',), 'Create a JPEG image from source code. This uses the Python Imaging Library to generate a pixmap from the source code.'),
'LatexFormatter': ('pygments.formatters.latex', 'LaTeX', ('latex', 'tex'), ('*.tex',), 'Format tokens as LaTeX code. This needs the `fancyvrb` and `color` standard packages.'),
'NullFormatter': ('pygments.formatters.other', 'Text only', ('text', 'null'), ('*.txt',), 'Output the text unchanged without any formatting.'),
'RawTokenFormatter': ('pygments.formatters.other', 'Raw tokens', ('raw', 'tokens'), ('*.raw',), 'Format tokens as a raw representation for storing token streams.'),
'RtfFormatter': ('pygments.formatters.rtf', 'RTF', ('rtf',), ('*.rtf',), 'Format tokens as RTF markup. This formatter automatically outputs full RTF documents with color information and other useful stuff. Perfect for Copy and Paste into Microsoft(R) Word(R) documents.'),
'SvgFormatter': ('pygments.formatters.svg', 'SVG', ('svg',), ('*.svg',), 'Format tokens as an SVG graphics file. This formatter is still experimental. Each line of code is a ``<text>`` element with explicit ``x`` and ``y`` coordinates containing ``<tspan>`` elements with the individual token styles.'),
'Terminal256Formatter': ('pygments.formatters.terminal256', 'Terminal256', ('terminal256', 'console256', '256'), (), 'Format tokens with ANSI color sequences, for output in a 256-color terminal or console. Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'),
'TerminalFormatter': ('pygments.formatters.terminal', 'Terminal', ('terminal', 'console'), (), 'Format tokens with ANSI color sequences, for output in a text console. Color sequences are terminated at newlines, so that paging the output works correctly.'),
'TerminalTrueColorFormatter': ('pygments.formatters.terminal256', 'TerminalTrueColor', ('terminal16m', 'console16m', '16m'), (), 'Format tokens with ANSI color sequences, for output in a true-color terminal or console. Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'),
'TestcaseFormatter': ('pygments.formatters.other', 'Testcase', ('testcase',), (), 'Format tokens as appropriate for a new testcase.')
}
if __name__ == '__main__': # pragma: no cover
import sys
import os
# lookup formatters
found_formatters = []
imports = []
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
from pygments.util import docstring_headline
for root, dirs, files in os.walk('.'):
for filename in files:
if filename.endswith('.py') and not filename.startswith('_'):
module_name = 'pygments.formatters%s.%s' % (
root[1:].replace('/', '.'), filename[:-3])
print(module_name)
module = __import__(module_name, None, None, [''])
for formatter_name in module.__all__:
formatter = getattr(module, formatter_name)
found_formatters.append(
'%r: %r' % (formatter_name,
(module_name,
formatter.name,
tuple(formatter.aliases),
tuple(formatter.filenames),
docstring_headline(formatter))))
# sort them to make the diff minimal
found_formatters.sort()
# extract useful sourcecode from this file
with open(__file__) as fp:
content = fp.read()
# replace crnl to nl for Windows.
#
# Note that, originally, contributers should keep nl of master
# repository, for example by using some kind of automatic
# management EOL, like `EolExtension
# <https://www.mercurial-scm.org/wiki/EolExtension>`.
content = content.replace("\r\n", "\n")
header = content[:content.find('FORMATTERS = {')]
footer = content[content.find("if __name__ == '__main__':"):]
# write new file
with open(__file__, 'w') as fp:
fp.write(header)
fp.write('FORMATTERS = {\n %s\n}\n\n' % ',\n '.join(found_formatters))
fp.write(footer)
print ('=== %d formatters processed.' % len(found_formatters))

View File

@ -1,109 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.formatters.bbcode
~~~~~~~~~~~~~~~~~~~~~~~~~~
BBcode formatter.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from pygments.formatter import Formatter
from pygments.util import get_bool_opt
__all__ = ['BBCodeFormatter']
class BBCodeFormatter(Formatter):
"""
Format tokens with BBcodes. These formatting codes are used by many
bulletin boards, so you can highlight your sourcecode with pygments before
posting it there.
This formatter has no support for background colors and borders, as there
are no common BBcode tags for that.
Some board systems (e.g. phpBB) don't support colors in their [code] tag,
so you can't use the highlighting together with that tag.
Text in a [code] tag usually is shown with a monospace font (which this
formatter can do with the ``monofont`` option) and no spaces (which you
need for indentation) are removed.
Additional options accepted:
`style`
The style to use, can be a string or a Style subclass (default:
``'default'``).
`codetag`
If set to true, put the output into ``[code]`` tags (default:
``false``)
`monofont`
If set to true, add a tag to show the code with a monospace font
(default: ``false``).
"""
name = 'BBCode'
aliases = ['bbcode', 'bb']
filenames = []
def __init__(self, **options):
Formatter.__init__(self, **options)
self._code = get_bool_opt(options, 'codetag', False)
self._mono = get_bool_opt(options, 'monofont', False)
self.styles = {}
self._make_styles()
def _make_styles(self):
for ttype, ndef in self.style:
start = end = ''
if ndef['color']:
start += '[color=#%s]' % ndef['color']
end = '[/color]' + end
if ndef['bold']:
start += '[b]'
end = '[/b]' + end
if ndef['italic']:
start += '[i]'
end = '[/i]' + end
if ndef['underline']:
start += '[u]'
end = '[/u]' + end
# there are no common BBcodes for background-color and border
self.styles[ttype] = start, end
def format_unencoded(self, tokensource, outfile):
if self._code:
outfile.write('[code]')
if self._mono:
outfile.write('[font=monospace]')
lastval = ''
lasttype = None
for ttype, value in tokensource:
while ttype not in self.styles:
ttype = ttype.parent
if ttype == lasttype:
lastval += value
else:
if lastval:
start, end = self.styles[lasttype]
outfile.write(''.join((start, lastval, end)))
lastval = value
lasttype = ttype
if lastval:
start, end = self.styles[lasttype]
outfile.write(''.join((start, lastval, end)))
if self._mono:
outfile.write('[/font]')
if self._code:
outfile.write('[/code]')
if self._code or self._mono:
outfile.write('\n')

View File

@ -1,851 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.formatters.html
~~~~~~~~~~~~~~~~~~~~~~~~
Formatter for HTML output.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from __future__ import print_function
import os
import sys
import os.path
from pygments.formatter import Formatter
from pygments.token import Token, Text, STANDARD_TYPES
from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
StringIO, string_types, iteritems
try:
import ctags
except ImportError:
ctags = None
__all__ = ['HtmlFormatter']
_escape_html_table = {
ord('&'): u'&amp;',
ord('<'): u'&lt;',
ord('>'): u'&gt;',
ord('"'): u'&quot;',
ord("'"): u'&#39;',
}
def escape_html(text, table=_escape_html_table):
"""Escape &, <, > as well as single and double quotes for HTML."""
return text.translate(table)
def _get_ttype_class(ttype):
fname = STANDARD_TYPES.get(ttype)
if fname:
return fname
aname = ''
while fname is None:
aname = '-' + ttype[-1] + aname
ttype = ttype.parent
fname = STANDARD_TYPES.get(ttype)
return fname + aname
CSSFILE_TEMPLATE = '''\
td.linenos { background-color: #f0f0f0; padding-right: 10px; }
span.lineno { background-color: #f0f0f0; padding: 0 5px 0 5px; }
pre { line-height: 125%%; }
%(styledefs)s
'''
DOC_HEADER = '''\
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"
"http://www.w3.org/TR/html4/strict.dtd">
<html>
<head>
<title>%(title)s</title>
<meta http-equiv="content-type" content="text/html; charset=%(encoding)s">
<style type="text/css">
''' + CSSFILE_TEMPLATE + '''
</style>
</head>
<body>
<h2>%(title)s</h2>
'''
DOC_HEADER_EXTERNALCSS = '''\
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"
"http://www.w3.org/TR/html4/strict.dtd">
<html>
<head>
<title>%(title)s</title>
<meta http-equiv="content-type" content="text/html; charset=%(encoding)s">
<link rel="stylesheet" href="%(cssfile)s" type="text/css">
</head>
<body>
<h2>%(title)s</h2>
'''
DOC_FOOTER = '''\
</body>
</html>
'''
class HtmlFormatter(Formatter):
r"""
Format tokens as HTML 4 ``<span>`` tags within a ``<pre>`` tag, wrapped
in a ``<div>`` tag. The ``<div>``'s CSS class can be set by the `cssclass`
option.
If the `linenos` option is set to ``"table"``, the ``<pre>`` is
additionally wrapped inside a ``<table>`` which has one row and two
cells: one containing the line numbers and one containing the code.
Example:
.. sourcecode:: html
<div class="highlight" >
<table><tr>
<td class="linenos" title="click to toggle"
onclick="with (this.firstChild.style)
{ display = (display == '') ? 'none' : '' }">
<pre>1
2</pre>
</td>
<td class="code">
<pre><span class="Ke">def </span><span class="NaFu">foo</span>(bar):
<span class="Ke">pass</span>
</pre>
</td>
</tr></table></div>
(whitespace added to improve clarity).
Wrapping can be disabled using the `nowrap` option.
A list of lines can be specified using the `hl_lines` option to make these
lines highlighted (as of Pygments 0.11).
With the `full` option, a complete HTML 4 document is output, including
the style definitions inside a ``<style>`` tag, or in a separate file if
the `cssfile` option is given.
When `tagsfile` is set to the path of a ctags index file, it is used to
generate hyperlinks from names to their definition. You must enable
`lineanchors` and run ctags with the `-n` option for this to work. The
`python-ctags` module from PyPI must be installed to use this feature;
otherwise a `RuntimeError` will be raised.
The `get_style_defs(arg='')` method of a `HtmlFormatter` returns a string
containing CSS rules for the CSS classes used by the formatter. The
argument `arg` can be used to specify additional CSS selectors that
are prepended to the classes. A call `fmter.get_style_defs('td .code')`
would result in the following CSS classes:
.. sourcecode:: css
td .code .kw { font-weight: bold; color: #00FF00 }
td .code .cm { color: #999999 }
...
If you have Pygments 0.6 or higher, you can also pass a list or tuple to the
`get_style_defs()` method to request multiple prefixes for the tokens:
.. sourcecode:: python
formatter.get_style_defs(['div.syntax pre', 'pre.syntax'])
The output would then look like this:
.. sourcecode:: css
div.syntax pre .kw,
pre.syntax .kw { font-weight: bold; color: #00FF00 }
div.syntax pre .cm,
pre.syntax .cm { color: #999999 }
...
Additional options accepted:
`nowrap`
If set to ``True``, don't wrap the tokens at all, not even inside a ``<pre>``
tag. This disables most other options (default: ``False``).
`full`
Tells the formatter to output a "full" document, i.e. a complete
self-contained document (default: ``False``).
`title`
If `full` is true, the title that should be used to caption the
document (default: ``''``).
`style`
The style to use, can be a string or a Style subclass (default:
``'default'``). This option has no effect if the `cssfile`
and `noclobber_cssfile` option are given and the file specified in
`cssfile` exists.
`noclasses`
If set to true, token ``<span>`` tags will not use CSS classes, but
inline styles. This is not recommended for larger pieces of code since
it increases output size by quite a bit (default: ``False``).
`classprefix`
Since the token types use relatively short class names, they may clash
with some of your own class names. In this case you can use the
`classprefix` option to give a string to prepend to all Pygments-generated
CSS class names for token types.
Note that this option also affects the output of `get_style_defs()`.
`cssclass`
CSS class for the wrapping ``<div>`` tag (default: ``'highlight'``).
If you set this option, the default selector for `get_style_defs()`
will be this class.
.. versionadded:: 0.9
If you select the ``'table'`` line numbers, the wrapping table will
have a CSS class of this string plus ``'table'``, the default is
accordingly ``'highlighttable'``.
`cssstyles`
Inline CSS styles for the wrapping ``<div>`` tag (default: ``''``).
`prestyles`
Inline CSS styles for the ``<pre>`` tag (default: ``''``).
.. versionadded:: 0.11
`cssfile`
If the `full` option is true and this option is given, it must be the
name of an external file. If the filename does not include an absolute
path, the file's path will be assumed to be relative to the main output
file's path, if the latter can be found. The stylesheet is then written
to this file instead of the HTML file.
.. versionadded:: 0.6
`noclobber_cssfile`
If `cssfile` is given and the specified file exists, the css file will
not be overwritten. This allows the use of the `full` option in
combination with a user specified css file. Default is ``False``.
.. versionadded:: 1.1
`linenos`
If set to ``'table'``, output line numbers as a table with two cells,
one containing the line numbers, the other the whole code. This is
copy-and-paste-friendly, but may cause alignment problems with some
browsers or fonts. If set to ``'inline'``, the line numbers will be
integrated in the ``<pre>`` tag that contains the code (that setting
is *new in Pygments 0.8*).
For compatibility with Pygments 0.7 and earlier, every true value
except ``'inline'`` means the same as ``'table'`` (in particular, that
means also ``True``).
The default value is ``False``, which means no line numbers at all.
**Note:** with the default ("table") line number mechanism, the line
numbers and code can have different line heights in Internet Explorer
unless you give the enclosing ``<pre>`` tags an explicit ``line-height``
CSS property (you get the default line spacing with ``line-height:
125%``).
`hl_lines`
Specify a list of lines to be highlighted.
.. versionadded:: 0.11
`linenostart`
The line number for the first line (default: ``1``).
`linenostep`
If set to a number n > 1, only every nth line number is printed.
`linenospecial`
If set to a number n > 0, every nth line number is given the CSS
class ``"special"`` (default: ``0``).
`nobackground`
If set to ``True``, the formatter won't output the background color
for the wrapping element (this automatically defaults to ``False``
when there is no wrapping element [eg: no argument for the
`get_syntax_defs` method given]) (default: ``False``).
.. versionadded:: 0.6
`lineseparator`
This string is output between lines of code. It defaults to ``"\n"``,
which is enough to break a line inside ``<pre>`` tags, but you can
e.g. set it to ``"<br>"`` to get HTML line breaks.
.. versionadded:: 0.7
`lineanchors`
If set to a nonempty string, e.g. ``foo``, the formatter will wrap each
output line in an anchor tag with a ``name`` of ``foo-linenumber``.
This allows easy linking to certain lines.
.. versionadded:: 0.9
`linespans`
If set to a nonempty string, e.g. ``foo``, the formatter will wrap each
output line in a span tag with an ``id`` of ``foo-linenumber``.
This allows easy access to lines via javascript.
.. versionadded:: 1.6
`anchorlinenos`
If set to `True`, will wrap line numbers in <a> tags. Used in
combination with `linenos` and `lineanchors`.
`tagsfile`
If set to the path of a ctags file, wrap names in anchor tags that
link to their definitions. `lineanchors` should be used, and the
tags file should specify line numbers (see the `-n` option to ctags).
.. versionadded:: 1.6
`tagurlformat`
A string formatting pattern used to generate links to ctags definitions.
Available variables are `%(path)s`, `%(fname)s` and `%(fext)s`.
Defaults to an empty string, resulting in just `#prefix-number` links.
.. versionadded:: 1.6
`filename`
A string used to generate a filename when rendering <pre> blocks,
for example if displaying source code.
.. versionadded:: 2.1
**Subclassing the HTML formatter**
.. versionadded:: 0.7
The HTML formatter is now built in a way that allows easy subclassing, thus
customizing the output HTML code. The `format()` method calls
`self._format_lines()` which returns a generator that yields tuples of ``(1,
line)``, where the ``1`` indicates that the ``line`` is a line of the
formatted source code.
If the `nowrap` option is set, the generator is the iterated over and the
resulting HTML is output.
Otherwise, `format()` calls `self.wrap()`, which wraps the generator with
other generators. These may add some HTML code to the one generated by
`_format_lines()`, either by modifying the lines generated by the latter,
then yielding them again with ``(1, line)``, and/or by yielding other HTML
code before or after the lines, with ``(0, html)``. The distinction between
source lines and other code makes it possible to wrap the generator multiple
times.
The default `wrap()` implementation adds a ``<div>`` and a ``<pre>`` tag.
A custom `HtmlFormatter` subclass could look like this:
.. sourcecode:: python
class CodeHtmlFormatter(HtmlFormatter):
def wrap(self, source, outfile):
return self._wrap_code(source)
def _wrap_code(self, source):
yield 0, '<code>'
for i, t in source:
if i == 1:
# it's a line of formatted code
t += '<br>'
yield i, t
yield 0, '</code>'
This results in wrapping the formatted lines with a ``<code>`` tag, where the
source lines are broken using ``<br>`` tags.
After calling `wrap()`, the `format()` method also adds the "line numbers"
and/or "full document" wrappers if the respective options are set. Then, all
HTML yielded by the wrapped generator is output.
"""
name = 'HTML'
aliases = ['html']
filenames = ['*.html', '*.htm']
def __init__(self, **options):
Formatter.__init__(self, **options)
self.title = self._decodeifneeded(self.title)
self.nowrap = get_bool_opt(options, 'nowrap', False)
self.noclasses = get_bool_opt(options, 'noclasses', False)
self.classprefix = options.get('classprefix', '')
self.cssclass = self._decodeifneeded(options.get('cssclass', 'highlight'))
self.cssstyles = self._decodeifneeded(options.get('cssstyles', ''))
self.prestyles = self._decodeifneeded(options.get('prestyles', ''))
self.cssfile = self._decodeifneeded(options.get('cssfile', ''))
self.noclobber_cssfile = get_bool_opt(options, 'noclobber_cssfile', False)
self.tagsfile = self._decodeifneeded(options.get('tagsfile', ''))
self.tagurlformat = self._decodeifneeded(options.get('tagurlformat', ''))
self.filename = self._decodeifneeded(options.get('filename', ''))
if self.tagsfile:
if not ctags:
raise RuntimeError('The "ctags" package must to be installed '
'to be able to use the "tagsfile" feature.')
self._ctags = ctags.CTags(self.tagsfile)
linenos = options.get('linenos', False)
if linenos == 'inline':
self.linenos = 2
elif linenos:
# compatibility with <= 0.7
self.linenos = 1
else:
self.linenos = 0
self.linenostart = abs(get_int_opt(options, 'linenostart', 1))
self.linenostep = abs(get_int_opt(options, 'linenostep', 1))
self.linenospecial = abs(get_int_opt(options, 'linenospecial', 0))
self.nobackground = get_bool_opt(options, 'nobackground', False)
self.lineseparator = options.get('lineseparator', '\n')
self.lineanchors = options.get('lineanchors', '')
self.linespans = options.get('linespans', '')
self.anchorlinenos = options.get('anchorlinenos', False)
self.hl_lines = set()
for lineno in get_list_opt(options, 'hl_lines', []):
try:
self.hl_lines.add(int(lineno))
except ValueError:
pass
self._create_stylesheet()
def _get_css_class(self, ttype):
"""Return the css class of this token type prefixed with
the classprefix option."""
ttypeclass = _get_ttype_class(ttype)
if ttypeclass:
return self.classprefix + ttypeclass
return ''
def _get_css_classes(self, ttype):
"""Return the css classes of this token type prefixed with
the classprefix option."""
cls = self._get_css_class(ttype)
while ttype not in STANDARD_TYPES:
ttype = ttype.parent
cls = self._get_css_class(ttype) + ' ' + cls
return cls
def _create_stylesheet(self):
t2c = self.ttype2class = {Token: ''}
c2s = self.class2style = {}
for ttype, ndef in self.style:
name = self._get_css_class(ttype)
style = ''
if ndef['color']:
style += 'color: #%s; ' % ndef['color']
if ndef['bold']:
style += 'font-weight: bold; '
if ndef['italic']:
style += 'font-style: italic; '
if ndef['underline']:
style += 'text-decoration: underline; '
if ndef['bgcolor']:
style += 'background-color: #%s; ' % ndef['bgcolor']
if ndef['border']:
style += 'border: 1px solid #%s; ' % ndef['border']
if style:
t2c[ttype] = name
# save len(ttype) to enable ordering the styles by
# hierarchy (necessary for CSS cascading rules!)
c2s[name] = (style[:-2], ttype, len(ttype))
def get_style_defs(self, arg=None):
"""
Return CSS style definitions for the classes produced by the current
highlighting style. ``arg`` can be a string or list of selectors to
insert before the token type classes.
"""
if arg is None:
arg = ('cssclass' in self.options and '.'+self.cssclass or '')
if isinstance(arg, string_types):
args = [arg]
else:
args = list(arg)
def prefix(cls):
if cls:
cls = '.' + cls
tmp = []
for arg in args:
tmp.append((arg and arg + ' ' or '') + cls)
return ', '.join(tmp)
styles = [(level, ttype, cls, style)
for cls, (style, ttype, level) in iteritems(self.class2style)
if cls and style]
styles.sort()
lines = ['%s { %s } /* %s */' % (prefix(cls), style, repr(ttype)[6:])
for (level, ttype, cls, style) in styles]
if arg and not self.nobackground and \
self.style.background_color is not None:
text_style = ''
if Text in self.ttype2class:
text_style = ' ' + self.class2style[self.ttype2class[Text]][0]
lines.insert(0, '%s { background: %s;%s }' %
(prefix(''), self.style.background_color, text_style))
if self.style.highlight_color is not None:
lines.insert(0, '%s.hll { background-color: %s }' %
(prefix(''), self.style.highlight_color))
return '\n'.join(lines)
def _decodeifneeded(self, value):
if isinstance(value, bytes):
if self.encoding:
return value.decode(self.encoding)
return value.decode()
return value
def _wrap_full(self, inner, outfile):
if self.cssfile:
if os.path.isabs(self.cssfile):
# it's an absolute filename
cssfilename = self.cssfile
else:
try:
filename = outfile.name
if not filename or filename[0] == '<':
# pseudo files, e.g. name == '<fdopen>'
raise AttributeError
cssfilename = os.path.join(os.path.dirname(filename),
self.cssfile)
except AttributeError:
print('Note: Cannot determine output file name, '
'using current directory as base for the CSS file name',
file=sys.stderr)
cssfilename = self.cssfile
# write CSS file only if noclobber_cssfile isn't given as an option.
try:
if not os.path.exists(cssfilename) or not self.noclobber_cssfile:
cf = open(cssfilename, "w")
cf.write(CSSFILE_TEMPLATE %
{'styledefs': self.get_style_defs('body')})
cf.close()
except IOError as err:
err.strerror = 'Error writing CSS file: ' + err.strerror
raise
yield 0, (DOC_HEADER_EXTERNALCSS %
dict(title=self.title,
cssfile=self.cssfile,
encoding=self.encoding))
else:
yield 0, (DOC_HEADER %
dict(title=self.title,
styledefs=self.get_style_defs('body'),
encoding=self.encoding))
for t, line in inner:
yield t, line
yield 0, DOC_FOOTER
def _wrap_tablelinenos(self, inner):
dummyoutfile = StringIO()
lncount = 0
for t, line in inner:
if t:
lncount += 1
dummyoutfile.write(line)
fl = self.linenostart
mw = len(str(lncount + fl - 1))
sp = self.linenospecial
st = self.linenostep
la = self.lineanchors
aln = self.anchorlinenos
nocls = self.noclasses
if sp:
lines = []
for i in range(fl, fl+lncount):
if i % st == 0:
if i % sp == 0:
if aln:
lines.append('<a href="#%s-%d" class="special">%*d</a>' %
(la, i, mw, i))
else:
lines.append('<span class="special">%*d</span>' % (mw, i))
else:
if aln:
lines.append('<a href="#%s-%d">%*d</a>' % (la, i, mw, i))
else:
lines.append('%*d' % (mw, i))
else:
lines.append('')
ls = '\n'.join(lines)
else:
lines = []
for i in range(fl, fl+lncount):
if i % st == 0:
if aln:
lines.append('<a href="#%s-%d">%*d</a>' % (la, i, mw, i))
else:
lines.append('%*d' % (mw, i))
else:
lines.append('')
ls = '\n'.join(lines)
# in case you wonder about the seemingly redundant <div> here: since the
# content in the other cell also is wrapped in a div, some browsers in
# some configurations seem to mess up the formatting...
if nocls:
yield 0, ('<table class="%stable">' % self.cssclass +
'<tr><td><div class="linenodiv" '
'style="background-color: #f0f0f0; padding-right: 10px">'
'<pre style="line-height: 125%">' +
ls + '</pre></div></td><td class="code">')
else:
yield 0, ('<table class="%stable">' % self.cssclass +
'<tr><td class="linenos"><div class="linenodiv"><pre>' +
ls + '</pre></div></td><td class="code">')
yield 0, dummyoutfile.getvalue()
yield 0, '</td></tr></table>'
def _wrap_inlinelinenos(self, inner):
# need a list of lines since we need the width of a single number :(
lines = list(inner)
sp = self.linenospecial
st = self.linenostep
num = self.linenostart
mw = len(str(len(lines) + num - 1))
if self.noclasses:
if sp:
for t, line in lines:
if num % sp == 0:
style = 'background-color: #ffffc0; padding: 0 5px 0 5px'
else:
style = 'background-color: #f0f0f0; padding: 0 5px 0 5px'
yield 1, '<span style="%s">%*s </span>' % (
style, mw, (num % st and ' ' or num)) + line
num += 1
else:
for t, line in lines:
yield 1, ('<span style="background-color: #f0f0f0; '
'padding: 0 5px 0 5px">%*s </span>' % (
mw, (num % st and ' ' or num)) + line)
num += 1
elif sp:
for t, line in lines:
yield 1, '<span class="lineno%s">%*s </span>' % (
num % sp == 0 and ' special' or '', mw,
(num % st and ' ' or num)) + line
num += 1
else:
for t, line in lines:
yield 1, '<span class="lineno">%*s </span>' % (
mw, (num % st and ' ' or num)) + line
num += 1
def _wrap_lineanchors(self, inner):
s = self.lineanchors
# subtract 1 since we have to increment i *before* yielding
i = self.linenostart - 1
for t, line in inner:
if t:
i += 1
yield 1, '<a name="%s-%d"></a>' % (s, i) + line
else:
yield 0, line
def _wrap_linespans(self, inner):
s = self.linespans
i = self.linenostart - 1
for t, line in inner:
if t:
i += 1
yield 1, '<span id="%s-%d">%s</span>' % (s, i, line)
else:
yield 0, line
def _wrap_div(self, inner):
style = []
if (self.noclasses and not self.nobackground and
self.style.background_color is not None):
style.append('background: %s' % (self.style.background_color,))
if self.cssstyles:
style.append(self.cssstyles)
style = '; '.join(style)
yield 0, ('<div' + (self.cssclass and ' class="%s"' % self.cssclass) +
(style and (' style="%s"' % style)) + '>')
for tup in inner:
yield tup
yield 0, '</div>\n'
def _wrap_pre(self, inner):
style = []
if self.prestyles:
style.append(self.prestyles)
if self.noclasses:
style.append('line-height: 125%')
style = '; '.join(style)
if self.filename:
yield 0, ('<span class="filename">' + self.filename + '</span>')
# the empty span here is to keep leading empty lines from being
# ignored by HTML parsers
yield 0, ('<pre' + (style and ' style="%s"' % style) + '><span></span>')
for tup in inner:
yield tup
yield 0, '</pre>'
def _format_lines(self, tokensource):
"""
Just format the tokens, without any wrapping tags.
Yield individual lines.
"""
nocls = self.noclasses
lsep = self.lineseparator
# for <span style=""> lookup only
getcls = self.ttype2class.get
c2s = self.class2style
escape_table = _escape_html_table
tagsfile = self.tagsfile
lspan = ''
line = []
for ttype, value in tokensource:
if nocls:
cclass = getcls(ttype)
while cclass is None:
ttype = ttype.parent
cclass = getcls(ttype)
cspan = cclass and '<span style="%s">' % c2s[cclass][0] or ''
else:
cls = self._get_css_classes(ttype)
cspan = cls and '<span class="%s">' % cls or ''
parts = value.translate(escape_table).split('\n')
if tagsfile and ttype in Token.Name:
filename, linenumber = self._lookup_ctag(value)
if linenumber:
base, filename = os.path.split(filename)
if base:
base += '/'
filename, extension = os.path.splitext(filename)
url = self.tagurlformat % {'path': base, 'fname': filename,
'fext': extension}
parts[0] = "<a href=\"%s#%s-%d\">%s" % \
(url, self.lineanchors, linenumber, parts[0])
parts[-1] = parts[-1] + "</a>"
# for all but the last line
for part in parts[:-1]:
if line:
if lspan != cspan:
line.extend(((lspan and '</span>'), cspan, part,
(cspan and '</span>'), lsep))
else: # both are the same
line.extend((part, (lspan and '</span>'), lsep))
yield 1, ''.join(line)
line = []
elif part:
yield 1, ''.join((cspan, part, (cspan and '</span>'), lsep))
else:
yield 1, lsep
# for the last line
if line and parts[-1]:
if lspan != cspan:
line.extend(((lspan and '</span>'), cspan, parts[-1]))
lspan = cspan
else:
line.append(parts[-1])
elif parts[-1]:
line = [cspan, parts[-1]]
lspan = cspan
# else we neither have to open a new span nor set lspan
if line:
line.extend(((lspan and '</span>'), lsep))
yield 1, ''.join(line)
def _lookup_ctag(self, token):
entry = ctags.TagEntry()
if self._ctags.find(entry, token, 0):
return entry['file'], entry['lineNumber']
else:
return None, None
def _highlight_lines(self, tokensource):
"""
Highlighted the lines specified in the `hl_lines` option by
post-processing the token stream coming from `_format_lines`.
"""
hls = self.hl_lines
for i, (t, value) in enumerate(tokensource):
if t != 1:
yield t, value
if i + 1 in hls: # i + 1 because Python indexes start at 0
if self.noclasses:
style = ''
if self.style.highlight_color is not None:
style = (' style="background-color: %s"' %
(self.style.highlight_color,))
yield 1, '<span%s>%s</span>' % (style, value)
else:
yield 1, '<span class="hll">%s</span>' % value
else:
yield 1, value
def wrap(self, source, outfile):
"""
Wrap the ``source``, which is a generator yielding
individual lines, in custom generators. See docstring
for `format`. Can be overridden.
"""
return self._wrap_div(self._wrap_pre(source))
def format_unencoded(self, tokensource, outfile):
"""
The formatting process uses several nested generators; which of
them are used is determined by the user's options.
Each generator should take at least one argument, ``inner``,
and wrap the pieces of text generated by this.
Always yield 2-tuples: (code, text). If "code" is 1, the text
is part of the original tokensource being highlighted, if it's
0, the text is some piece of wrapping. This makes it possible to
use several different wrappers that process the original source
linewise, e.g. line number generators.
"""
source = self._format_lines(tokensource)
if self.hl_lines:
source = self._highlight_lines(source)
if not self.nowrap:
if self.linenos == 2:
source = self._wrap_inlinelinenos(source)
if self.lineanchors:
source = self._wrap_lineanchors(source)
if self.linespans:
source = self._wrap_linespans(source)
source = self.wrap(source, outfile)
if self.linenos == 1:
source = self._wrap_tablelinenos(source)
if self.full:
source = self._wrap_full(source, outfile)
for t, piece in source:
outfile.write(piece)

View File

@ -1,600 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.formatters.img
~~~~~~~~~~~~~~~~~~~~~~~
Formatter for Pixmap output.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import os
import sys
from pygments.formatter import Formatter
from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
get_choice_opt, xrange
import subprocess
# Import this carefully
try:
from PIL import Image, ImageDraw, ImageFont
pil_available = True
except ImportError:
pil_available = False
try:
import _winreg
except ImportError:
try:
import winreg as _winreg
except ImportError:
_winreg = None
__all__ = ['ImageFormatter', 'GifImageFormatter', 'JpgImageFormatter',
'BmpImageFormatter']
# For some unknown reason every font calls it something different
STYLES = {
'NORMAL': ['', 'Roman', 'Book', 'Normal', 'Regular', 'Medium'],
'ITALIC': ['Oblique', 'Italic'],
'BOLD': ['Bold'],
'BOLDITALIC': ['Bold Oblique', 'Bold Italic'],
}
# A sane default for modern systems
DEFAULT_FONT_NAME_NIX = 'Bitstream Vera Sans Mono'
DEFAULT_FONT_NAME_WIN = 'Courier New'
DEFAULT_FONT_NAME_MAC = 'Courier New'
class PilNotAvailable(ImportError):
"""When Python imaging library is not available"""
class FontNotFound(Exception):
"""When there are no usable fonts specified"""
class FontManager(object):
"""
Manages a set of fonts: normal, italic, bold, etc...
"""
def __init__(self, font_name, font_size=14):
self.font_name = font_name
self.font_size = font_size
self.fonts = {}
self.encoding = None
if sys.platform.startswith('win'):
if not font_name:
self.font_name = DEFAULT_FONT_NAME_WIN
self._create_win()
elif sys.platform.startswith('darwin'):
if not font_name:
self.font_name = DEFAULT_FONT_NAME_MAC
self._create_mac()
else:
if not font_name:
self.font_name = DEFAULT_FONT_NAME_NIX
self._create_nix()
def _get_nix_font_path(self, name, style):
proc = subprocess.Popen(['fc-list', "%s:style=%s" % (name, style), 'file'],
stdout=subprocess.PIPE, stderr=None)
stdout, _ = proc.communicate()
if proc.returncode == 0:
lines = stdout.splitlines()
for line in lines:
if line.startswith(b'Fontconfig warning:'):
continue
path = line.decode().strip().strip(':')
if path:
return path
return None
def _create_nix(self):
for name in STYLES['NORMAL']:
path = self._get_nix_font_path(self.font_name, name)
if path is not None:
self.fonts['NORMAL'] = ImageFont.truetype(path, self.font_size)
break
else:
raise FontNotFound('No usable fonts named: "%s"' %
self.font_name)
for style in ('ITALIC', 'BOLD', 'BOLDITALIC'):
for stylename in STYLES[style]:
path = self._get_nix_font_path(self.font_name, stylename)
if path is not None:
self.fonts[style] = ImageFont.truetype(path, self.font_size)
break
else:
if style == 'BOLDITALIC':
self.fonts[style] = self.fonts['BOLD']
else:
self.fonts[style] = self.fonts['NORMAL']
def _get_mac_font_path(self, font_map, name, style):
return font_map.get((name + ' ' + style).strip().lower())
def _create_mac(self):
font_map = {}
for font_dir in (os.path.join(os.getenv("HOME"), 'Library/Fonts/'),
'/Library/Fonts/', '/System/Library/Fonts/'):
font_map.update(
((os.path.splitext(f)[0].lower(), os.path.join(font_dir, f))
for f in os.listdir(font_dir) if f.lower().endswith('ttf')))
for name in STYLES['NORMAL']:
path = self._get_mac_font_path(font_map, self.font_name, name)
if path is not None:
self.fonts['NORMAL'] = ImageFont.truetype(path, self.font_size)
break
else:
raise FontNotFound('No usable fonts named: "%s"' %
self.font_name)
for style in ('ITALIC', 'BOLD', 'BOLDITALIC'):
for stylename in STYLES[style]:
path = self._get_mac_font_path(font_map, self.font_name, stylename)
if path is not None:
self.fonts[style] = ImageFont.truetype(path, self.font_size)
break
else:
if style == 'BOLDITALIC':
self.fonts[style] = self.fonts['BOLD']
else:
self.fonts[style] = self.fonts['NORMAL']
def _lookup_win(self, key, basename, styles, fail=False):
for suffix in ('', ' (TrueType)'):
for style in styles:
try:
valname = '%s%s%s' % (basename, style and ' '+style, suffix)
val, _ = _winreg.QueryValueEx(key, valname)
return val
except EnvironmentError:
continue
else:
if fail:
raise FontNotFound('Font %s (%s) not found in registry' %
(basename, styles[0]))
return None
def _create_win(self):
try:
key = _winreg.OpenKey(
_winreg.HKEY_LOCAL_MACHINE,
r'Software\Microsoft\Windows NT\CurrentVersion\Fonts')
except EnvironmentError:
try:
key = _winreg.OpenKey(
_winreg.HKEY_LOCAL_MACHINE,
r'Software\Microsoft\Windows\CurrentVersion\Fonts')
except EnvironmentError:
raise FontNotFound('Can\'t open Windows font registry key')
try:
path = self._lookup_win(key, self.font_name, STYLES['NORMAL'], True)
self.fonts['NORMAL'] = ImageFont.truetype(path, self.font_size)
for style in ('ITALIC', 'BOLD', 'BOLDITALIC'):
path = self._lookup_win(key, self.font_name, STYLES[style])
if path:
self.fonts[style] = ImageFont.truetype(path, self.font_size)
else:
if style == 'BOLDITALIC':
self.fonts[style] = self.fonts['BOLD']
else:
self.fonts[style] = self.fonts['NORMAL']
finally:
_winreg.CloseKey(key)
def get_char_size(self):
"""
Get the character size.
"""
return self.fonts['NORMAL'].getsize('M')
def get_font(self, bold, oblique):
"""
Get the font based on bold and italic flags.
"""
if bold and oblique:
return self.fonts['BOLDITALIC']
elif bold:
return self.fonts['BOLD']
elif oblique:
return self.fonts['ITALIC']
else:
return self.fonts['NORMAL']
class ImageFormatter(Formatter):
"""
Create a PNG image from source code. This uses the Python Imaging Library to
generate a pixmap from the source code.
.. versionadded:: 0.10
Additional options accepted:
`image_format`
An image format to output to that is recognised by PIL, these include:
* "PNG" (default)
* "JPEG"
* "BMP"
* "GIF"
`line_pad`
The extra spacing (in pixels) between each line of text.
Default: 2
`font_name`
The font name to be used as the base font from which others, such as
bold and italic fonts will be generated. This really should be a
monospace font to look sane.
Default: "Bitstream Vera Sans Mono" on Windows, Courier New on \*nix
`font_size`
The font size in points to be used.
Default: 14
`image_pad`
The padding, in pixels to be used at each edge of the resulting image.
Default: 10
`line_numbers`
Whether line numbers should be shown: True/False
Default: True
`line_number_start`
The line number of the first line.
Default: 1
`line_number_step`
The step used when printing line numbers.
Default: 1
`line_number_bg`
The background colour (in "#123456" format) of the line number bar, or
None to use the style background color.
Default: "#eed"
`line_number_fg`
The text color of the line numbers (in "#123456"-like format).
Default: "#886"
`line_number_chars`
The number of columns of line numbers allowable in the line number
margin.
Default: 2
`line_number_bold`
Whether line numbers will be bold: True/False
Default: False
`line_number_italic`
Whether line numbers will be italicized: True/False
Default: False
`line_number_separator`
Whether a line will be drawn between the line number area and the
source code area: True/False
Default: True
`line_number_pad`
The horizontal padding (in pixels) between the line number margin, and
the source code area.
Default: 6
`hl_lines`
Specify a list of lines to be highlighted.
.. versionadded:: 1.2
Default: empty list
`hl_color`
Specify the color for highlighting lines.
.. versionadded:: 1.2
Default: highlight color of the selected style
"""
# Required by the pygments mapper
name = 'img'
aliases = ['img', 'IMG', 'png']
filenames = ['*.png']
unicodeoutput = False
default_image_format = 'png'
def __init__(self, **options):
"""
See the class docstring for explanation of options.
"""
if not pil_available:
raise PilNotAvailable(
'Python Imaging Library is required for this formatter')
Formatter.__init__(self, **options)
self.encoding = 'latin1' # let pygments.format() do the right thing
# Read the style
self.styles = dict(self.style)
if self.style.background_color is None:
self.background_color = '#fff'
else:
self.background_color = self.style.background_color
# Image options
self.image_format = get_choice_opt(
options, 'image_format', ['png', 'jpeg', 'gif', 'bmp'],
self.default_image_format, normcase=True)
self.image_pad = get_int_opt(options, 'image_pad', 10)
self.line_pad = get_int_opt(options, 'line_pad', 2)
# The fonts
fontsize = get_int_opt(options, 'font_size', 14)
self.fonts = FontManager(options.get('font_name', ''), fontsize)
self.fontw, self.fonth = self.fonts.get_char_size()
# Line number options
self.line_number_fg = options.get('line_number_fg', '#886')
self.line_number_bg = options.get('line_number_bg', '#eed')
self.line_number_chars = get_int_opt(options,
'line_number_chars', 2)
self.line_number_bold = get_bool_opt(options,
'line_number_bold', False)
self.line_number_italic = get_bool_opt(options,
'line_number_italic', False)
self.line_number_pad = get_int_opt(options, 'line_number_pad', 6)
self.line_numbers = get_bool_opt(options, 'line_numbers', True)
self.line_number_separator = get_bool_opt(options,
'line_number_separator', True)
self.line_number_step = get_int_opt(options, 'line_number_step', 1)
self.line_number_start = get_int_opt(options, 'line_number_start', 1)
if self.line_numbers:
self.line_number_width = (self.fontw * self.line_number_chars +
self.line_number_pad * 2)
else:
self.line_number_width = 0
self.hl_lines = []
hl_lines_str = get_list_opt(options, 'hl_lines', [])
for line in hl_lines_str:
try:
self.hl_lines.append(int(line))
except ValueError:
pass
self.hl_color = options.get('hl_color',
self.style.highlight_color) or '#f90'
self.drawables = []
def get_style_defs(self, arg=''):
raise NotImplementedError('The -S option is meaningless for the image '
'formatter. Use -O style=<stylename> instead.')
def _get_line_height(self):
"""
Get the height of a line.
"""
return self.fonth + self.line_pad
def _get_line_y(self, lineno):
"""
Get the Y coordinate of a line number.
"""
return lineno * self._get_line_height() + self.image_pad
def _get_char_width(self):
"""
Get the width of a character.
"""
return self.fontw
def _get_char_x(self, charno):
"""
Get the X coordinate of a character position.
"""
return charno * self.fontw + self.image_pad + self.line_number_width
def _get_text_pos(self, charno, lineno):
"""
Get the actual position for a character and line position.
"""
return self._get_char_x(charno), self._get_line_y(lineno)
def _get_linenumber_pos(self, lineno):
"""
Get the actual position for the start of a line number.
"""
return (self.image_pad, self._get_line_y(lineno))
def _get_text_color(self, style):
"""
Get the correct color for the token from the style.
"""
if style['color'] is not None:
fill = '#' + style['color']
else:
fill = '#000'
return fill
def _get_style_font(self, style):
"""
Get the correct font for the style.
"""
return self.fonts.get_font(style['bold'], style['italic'])
def _get_image_size(self, maxcharno, maxlineno):
"""
Get the required image size.
"""
return (self._get_char_x(maxcharno) + self.image_pad,
self._get_line_y(maxlineno + 0) + self.image_pad)
def _draw_linenumber(self, posno, lineno):
"""
Remember a line number drawable to paint later.
"""
self._draw_text(
self._get_linenumber_pos(posno),
str(lineno).rjust(self.line_number_chars),
font=self.fonts.get_font(self.line_number_bold,
self.line_number_italic),
fill=self.line_number_fg,
)
def _draw_text(self, pos, text, font, **kw):
"""
Remember a single drawable tuple to paint later.
"""
self.drawables.append((pos, text, font, kw))
def _create_drawables(self, tokensource):
"""
Create drawables for the token content.
"""
lineno = charno = maxcharno = 0
for ttype, value in tokensource:
while ttype not in self.styles:
ttype = ttype.parent
style = self.styles[ttype]
# TODO: make sure tab expansion happens earlier in the chain. It
# really ought to be done on the input, as to do it right here is
# quite complex.
value = value.expandtabs(4)
lines = value.splitlines(True)
# print lines
for i, line in enumerate(lines):
temp = line.rstrip('\n')
if temp:
self._draw_text(
self._get_text_pos(charno, lineno),
temp,
font = self._get_style_font(style),
fill = self._get_text_color(style)
)
charno += len(temp)
maxcharno = max(maxcharno, charno)
if line.endswith('\n'):
# add a line for each extra line in the value
charno = 0
lineno += 1
self.maxcharno = maxcharno
self.maxlineno = lineno
def _draw_line_numbers(self):
"""
Create drawables for the line numbers.
"""
if not self.line_numbers:
return
for p in xrange(self.maxlineno):
n = p + self.line_number_start
if (n % self.line_number_step) == 0:
self._draw_linenumber(p, n)
def _paint_line_number_bg(self, im):
"""
Paint the line number background on the image.
"""
if not self.line_numbers:
return
if self.line_number_fg is None:
return
draw = ImageDraw.Draw(im)
recth = im.size[-1]
rectw = self.image_pad + self.line_number_width - self.line_number_pad
draw.rectangle([(0, 0), (rectw, recth)],
fill=self.line_number_bg)
draw.line([(rectw, 0), (rectw, recth)], fill=self.line_number_fg)
del draw
def format(self, tokensource, outfile):
"""
Format ``tokensource``, an iterable of ``(tokentype, tokenstring)``
tuples and write it into ``outfile``.
This implementation calculates where it should draw each token on the
pixmap, then calculates the required pixmap size and draws the items.
"""
self._create_drawables(tokensource)
self._draw_line_numbers()
im = Image.new(
'RGB',
self._get_image_size(self.maxcharno, self.maxlineno),
self.background_color
)
self._paint_line_number_bg(im)
draw = ImageDraw.Draw(im)
# Highlight
if self.hl_lines:
x = self.image_pad + self.line_number_width - self.line_number_pad + 1
recth = self._get_line_height()
rectw = im.size[0] - x
for linenumber in self.hl_lines:
y = self._get_line_y(linenumber - 1)
draw.rectangle([(x, y), (x + rectw, y + recth)],
fill=self.hl_color)
for pos, value, font, kw in self.drawables:
draw.text(pos, value, font=font, **kw)
im.save(outfile, self.image_format.upper())
# Add one formatter per format, so that the "-f gif" option gives the correct result
# when used in pygmentize.
class GifImageFormatter(ImageFormatter):
"""
Create a GIF image from source code. This uses the Python Imaging Library to
generate a pixmap from the source code.
.. versionadded:: 1.0
"""
name = 'img_gif'
aliases = ['gif']
filenames = ['*.gif']
default_image_format = 'gif'
class JpgImageFormatter(ImageFormatter):
"""
Create a JPEG image from source code. This uses the Python Imaging Library to
generate a pixmap from the source code.
.. versionadded:: 1.0
"""
name = 'img_jpg'
aliases = ['jpg', 'jpeg']
filenames = ['*.jpg']
default_image_format = 'jpeg'
class BmpImageFormatter(ImageFormatter):
"""
Create a bitmap image from source code. This uses the Python Imaging Library to
generate a pixmap from the source code.
.. versionadded:: 1.0
"""
name = 'img_bmp'
aliases = ['bmp', 'bitmap']
filenames = ['*.bmp']
default_image_format = 'bmp'

View File

@ -1,182 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.formatters.irc
~~~~~~~~~~~~~~~~~~~~~~~
Formatter for IRC output
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import sys
from pygments.formatter import Formatter
from pygments.token import Keyword, Name, Comment, String, Error, \
Number, Operator, Generic, Token, Whitespace
from pygments.util import get_choice_opt
__all__ = ['IRCFormatter']
#: Map token types to a tuple of color values for light and dark
#: backgrounds.
IRC_COLORS = {
Token: ('', ''),
Whitespace: ('lightgray', 'darkgray'),
Comment: ('lightgray', 'darkgray'),
Comment.Preproc: ('teal', 'turquoise'),
Keyword: ('darkblue', 'blue'),
Keyword.Type: ('teal', 'turquoise'),
Operator.Word: ('purple', 'fuchsia'),
Name.Builtin: ('teal', 'turquoise'),
Name.Function: ('darkgreen', 'green'),
Name.Namespace: ('_teal_', '_turquoise_'),
Name.Class: ('_darkgreen_', '_green_'),
Name.Exception: ('teal', 'turquoise'),
Name.Decorator: ('darkgray', 'lightgray'),
Name.Variable: ('darkred', 'red'),
Name.Constant: ('darkred', 'red'),
Name.Attribute: ('teal', 'turquoise'),
Name.Tag: ('blue', 'blue'),
String: ('brown', 'brown'),
Number: ('darkblue', 'blue'),
Generic.Deleted: ('red', 'red'),
Generic.Inserted: ('darkgreen', 'green'),
Generic.Heading: ('**', '**'),
Generic.Subheading: ('*purple*', '*fuchsia*'),
Generic.Error: ('red', 'red'),
Error: ('_red_', '_red_'),
}
IRC_COLOR_MAP = {
'white': 0,
'black': 1,
'darkblue': 2,
'green': 3,
'red': 4,
'brown': 5,
'purple': 6,
'orange': 7,
'darkgreen': 7, #compat w/ ansi
'yellow': 8,
'lightgreen': 9,
'turquoise': 9, # compat w/ ansi
'teal': 10,
'lightblue': 11,
'darkred': 11, # compat w/ ansi
'blue': 12,
'fuchsia': 13,
'darkgray': 14,
'lightgray': 15,
}
def ircformat(color, text):
if len(color) < 1:
return text
add = sub = ''
if '_' in color: # italic
add += '\x1D'
sub = '\x1D' + sub
color = color.strip('_')
if '*' in color: # bold
add += '\x02'
sub = '\x02' + sub
color = color.strip('*')
# underline (\x1F) not supported
# backgrounds (\x03FF,BB) not supported
if len(color) > 0: # actual color - may have issues with ircformat("red", "blah")+"10" type stuff
add += '\x03' + str(IRC_COLOR_MAP[color]).zfill(2)
sub = '\x03' + sub
return add + text + sub
return '<'+add+'>'+text+'</'+sub+'>'
class IRCFormatter(Formatter):
r"""
Format tokens with IRC color sequences
The `get_style_defs()` method doesn't do anything special since there is
no support for common styles.
Options accepted:
`bg`
Set to ``"light"`` or ``"dark"`` depending on the terminal's background
(default: ``"light"``).
`colorscheme`
A dictionary mapping token types to (lightbg, darkbg) color names or
``None`` (default: ``None`` = use builtin colorscheme).
`linenos`
Set to ``True`` to have line numbers in the output as well
(default: ``False`` = no line numbers).
"""
name = 'IRC'
aliases = ['irc', 'IRC']
filenames = []
def __init__(self, **options):
Formatter.__init__(self, **options)
self.darkbg = get_choice_opt(options, 'bg',
['light', 'dark'], 'light') == 'dark'
self.colorscheme = options.get('colorscheme', None) or IRC_COLORS
self.linenos = options.get('linenos', False)
self._lineno = 0
def _write_lineno(self, outfile):
self._lineno += 1
outfile.write("\n%04d: " % self._lineno)
def _format_unencoded_with_lineno(self, tokensource, outfile):
self._write_lineno(outfile)
for ttype, value in tokensource:
if value.endswith("\n"):
self._write_lineno(outfile)
value = value[:-1]
color = self.colorscheme.get(ttype)
while color is None:
ttype = ttype[:-1]
color = self.colorscheme.get(ttype)
if color:
color = color[self.darkbg]
spl = value.split('\n')
for line in spl[:-1]:
self._write_lineno(outfile)
if line:
outfile.write(ircformat(color, line[:-1]))
if spl[-1]:
outfile.write(ircformat(color, spl[-1]))
else:
outfile.write(value)
outfile.write("\n")
def format_unencoded(self, tokensource, outfile):
if self.linenos:
self._format_unencoded_with_lineno(tokensource, outfile)
return
for ttype, value in tokensource:
color = self.colorscheme.get(ttype)
while color is None:
ttype = ttype[:-1]
color = self.colorscheme.get(ttype)
if color:
color = color[self.darkbg]
spl = value.split('\n')
for line in spl[:-1]:
if line:
outfile.write(ircformat(color, line))
outfile.write('\n')
if spl[-1]:
outfile.write(ircformat(color, spl[-1]))
else:
outfile.write(value)

View File

@ -1,482 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.formatters.latex
~~~~~~~~~~~~~~~~~~~~~~~~~
Formatter for LaTeX fancyvrb output.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from __future__ import division
from pygments.formatter import Formatter
from pygments.lexer import Lexer
from pygments.token import Token, STANDARD_TYPES
from pygments.util import get_bool_opt, get_int_opt, StringIO, xrange, \
iteritems
__all__ = ['LatexFormatter']
def escape_tex(text, commandprefix):
return text.replace('\\', '\x00'). \
replace('{', '\x01'). \
replace('}', '\x02'). \
replace('\x00', r'\%sZbs{}' % commandprefix). \
replace('\x01', r'\%sZob{}' % commandprefix). \
replace('\x02', r'\%sZcb{}' % commandprefix). \
replace('^', r'\%sZca{}' % commandprefix). \
replace('_', r'\%sZus{}' % commandprefix). \
replace('&', r'\%sZam{}' % commandprefix). \
replace('<', r'\%sZlt{}' % commandprefix). \
replace('>', r'\%sZgt{}' % commandprefix). \
replace('#', r'\%sZsh{}' % commandprefix). \
replace('%', r'\%sZpc{}' % commandprefix). \
replace('$', r'\%sZdl{}' % commandprefix). \
replace('-', r'\%sZhy{}' % commandprefix). \
replace("'", r'\%sZsq{}' % commandprefix). \
replace('"', r'\%sZdq{}' % commandprefix). \
replace('~', r'\%sZti{}' % commandprefix)
DOC_TEMPLATE = r'''
\documentclass{%(docclass)s}
\usepackage{fancyvrb}
\usepackage{color}
\usepackage[%(encoding)s]{inputenc}
%(preamble)s
%(styledefs)s
\begin{document}
\section*{%(title)s}
%(code)s
\end{document}
'''
## Small explanation of the mess below :)
#
# The previous version of the LaTeX formatter just assigned a command to
# each token type defined in the current style. That obviously is
# problematic if the highlighted code is produced for a different style
# than the style commands themselves.
#
# This version works much like the HTML formatter which assigns multiple
# CSS classes to each <span> tag, from the most specific to the least
# specific token type, thus falling back to the parent token type if one
# is not defined. Here, the classes are there too and use the same short
# forms given in token.STANDARD_TYPES.
#
# Highlighted code now only uses one custom command, which by default is
# \PY and selectable by the commandprefix option (and in addition the
# escapes \PYZat, \PYZlb and \PYZrb which haven't been renamed for
# backwards compatibility purposes).
#
# \PY has two arguments: the classes, separated by +, and the text to
# render in that style. The classes are resolved into the respective
# style commands by magic, which serves to ignore unknown classes.
#
# The magic macros are:
# * \PY@it, \PY@bf, etc. are unconditionally wrapped around the text
# to render in \PY@do. Their definition determines the style.
# * \PY@reset resets \PY@it etc. to do nothing.
# * \PY@toks parses the list of classes, using magic inspired by the
# keyval package (but modified to use plusses instead of commas
# because fancyvrb redefines commas inside its environments).
# * \PY@tok processes one class, calling the \PY@tok@classname command
# if it exists.
# * \PY@tok@classname sets the \PY@it etc. to reflect the chosen style
# for its class.
# * \PY resets the style, parses the classnames and then calls \PY@do.
#
# Tip: to read this code, print it out in substituted form using e.g.
# >>> print STYLE_TEMPLATE % {'cp': 'PY'}
STYLE_TEMPLATE = r'''
\makeatletter
\def\%(cp)s@reset{\let\%(cp)s@it=\relax \let\%(cp)s@bf=\relax%%
\let\%(cp)s@ul=\relax \let\%(cp)s@tc=\relax%%
\let\%(cp)s@bc=\relax \let\%(cp)s@ff=\relax}
\def\%(cp)s@tok#1{\csname %(cp)s@tok@#1\endcsname}
\def\%(cp)s@toks#1+{\ifx\relax#1\empty\else%%
\%(cp)s@tok{#1}\expandafter\%(cp)s@toks\fi}
\def\%(cp)s@do#1{\%(cp)s@bc{\%(cp)s@tc{\%(cp)s@ul{%%
\%(cp)s@it{\%(cp)s@bf{\%(cp)s@ff{#1}}}}}}}
\def\%(cp)s#1#2{\%(cp)s@reset\%(cp)s@toks#1+\relax+\%(cp)s@do{#2}}
%(styles)s
\def\%(cp)sZbs{\char`\\}
\def\%(cp)sZus{\char`\_}
\def\%(cp)sZob{\char`\{}
\def\%(cp)sZcb{\char`\}}
\def\%(cp)sZca{\char`\^}
\def\%(cp)sZam{\char`\&}
\def\%(cp)sZlt{\char`\<}
\def\%(cp)sZgt{\char`\>}
\def\%(cp)sZsh{\char`\#}
\def\%(cp)sZpc{\char`\%%}
\def\%(cp)sZdl{\char`\$}
\def\%(cp)sZhy{\char`\-}
\def\%(cp)sZsq{\char`\'}
\def\%(cp)sZdq{\char`\"}
\def\%(cp)sZti{\char`\~}
%% for compatibility with earlier versions
\def\%(cp)sZat{@}
\def\%(cp)sZlb{[}
\def\%(cp)sZrb{]}
\makeatother
'''
def _get_ttype_name(ttype):
fname = STANDARD_TYPES.get(ttype)
if fname:
return fname
aname = ''
while fname is None:
aname = ttype[-1] + aname
ttype = ttype.parent
fname = STANDARD_TYPES.get(ttype)
return fname + aname
class LatexFormatter(Formatter):
r"""
Format tokens as LaTeX code. This needs the `fancyvrb` and `color`
standard packages.
Without the `full` option, code is formatted as one ``Verbatim``
environment, like this:
.. sourcecode:: latex
\begin{Verbatim}[commandchars=\\\{\}]
\PY{k}{def }\PY{n+nf}{foo}(\PY{n}{bar}):
\PY{k}{pass}
\end{Verbatim}
The special command used here (``\PY``) and all the other macros it needs
are output by the `get_style_defs` method.
With the `full` option, a complete LaTeX document is output, including
the command definitions in the preamble.
The `get_style_defs()` method of a `LatexFormatter` returns a string
containing ``\def`` commands defining the macros needed inside the
``Verbatim`` environments.
Additional options accepted:
`style`
The style to use, can be a string or a Style subclass (default:
``'default'``).
`full`
Tells the formatter to output a "full" document, i.e. a complete
self-contained document (default: ``False``).
`title`
If `full` is true, the title that should be used to caption the
document (default: ``''``).
`docclass`
If the `full` option is enabled, this is the document class to use
(default: ``'article'``).
`preamble`
If the `full` option is enabled, this can be further preamble commands,
e.g. ``\usepackage`` (default: ``''``).
`linenos`
If set to ``True``, output line numbers (default: ``False``).
`linenostart`
The line number for the first line (default: ``1``).
`linenostep`
If set to a number n > 1, only every nth line number is printed.
`verboptions`
Additional options given to the Verbatim environment (see the *fancyvrb*
docs for possible values) (default: ``''``).
`commandprefix`
The LaTeX commands used to produce colored output are constructed
using this prefix and some letters (default: ``'PY'``).
.. versionadded:: 0.7
.. versionchanged:: 0.10
The default is now ``'PY'`` instead of ``'C'``.
`texcomments`
If set to ``True``, enables LaTeX comment lines. That is, LaTex markup
in comment tokens is not escaped so that LaTeX can render it (default:
``False``).
.. versionadded:: 1.2
`mathescape`
If set to ``True``, enables LaTeX math mode escape in comments. That
is, ``'$...$'`` inside a comment will trigger math mode (default:
``False``).
.. versionadded:: 1.2
`escapeinside`
If set to a string of length 2, enables escaping to LaTeX. Text
delimited by these 2 characters is read as LaTeX code and
typeset accordingly. It has no effect in string literals. It has
no effect in comments if `texcomments` or `mathescape` is
set. (default: ``''``).
.. versionadded:: 2.0
`envname`
Allows you to pick an alternative environment name replacing Verbatim.
The alternate environment still has to support Verbatim's option syntax.
(default: ``'Verbatim'``).
.. versionadded:: 2.0
"""
name = 'LaTeX'
aliases = ['latex', 'tex']
filenames = ['*.tex']
def __init__(self, **options):
Formatter.__init__(self, **options)
self.docclass = options.get('docclass', 'article')
self.preamble = options.get('preamble', '')
self.linenos = get_bool_opt(options, 'linenos', False)
self.linenostart = abs(get_int_opt(options, 'linenostart', 1))
self.linenostep = abs(get_int_opt(options, 'linenostep', 1))
self.verboptions = options.get('verboptions', '')
self.nobackground = get_bool_opt(options, 'nobackground', False)
self.commandprefix = options.get('commandprefix', 'PY')
self.texcomments = get_bool_opt(options, 'texcomments', False)
self.mathescape = get_bool_opt(options, 'mathescape', False)
self.escapeinside = options.get('escapeinside', '')
if len(self.escapeinside) == 2:
self.left = self.escapeinside[0]
self.right = self.escapeinside[1]
else:
self.escapeinside = ''
self.envname = options.get('envname', u'Verbatim')
self._create_stylesheet()
def _create_stylesheet(self):
t2n = self.ttype2name = {Token: ''}
c2d = self.cmd2def = {}
cp = self.commandprefix
def rgbcolor(col):
if col:
return ','.join(['%.2f' % (int(col[i] + col[i + 1], 16) / 255.0)
for i in (0, 2, 4)])
else:
return '1,1,1'
for ttype, ndef in self.style:
name = _get_ttype_name(ttype)
cmndef = ''
if ndef['bold']:
cmndef += r'\let\$$@bf=\textbf'
if ndef['italic']:
cmndef += r'\let\$$@it=\textit'
if ndef['underline']:
cmndef += r'\let\$$@ul=\underline'
if ndef['roman']:
cmndef += r'\let\$$@ff=\textrm'
if ndef['sans']:
cmndef += r'\let\$$@ff=\textsf'
if ndef['mono']:
cmndef += r'\let\$$@ff=\textsf'
if ndef['color']:
cmndef += (r'\def\$$@tc##1{\textcolor[rgb]{%s}{##1}}' %
rgbcolor(ndef['color']))
if ndef['border']:
cmndef += (r'\def\$$@bc##1{\setlength{\fboxsep}{0pt}'
r'\fcolorbox[rgb]{%s}{%s}{\strut ##1}}' %
(rgbcolor(ndef['border']),
rgbcolor(ndef['bgcolor'])))
elif ndef['bgcolor']:
cmndef += (r'\def\$$@bc##1{\setlength{\fboxsep}{0pt}'
r'\colorbox[rgb]{%s}{\strut ##1}}' %
rgbcolor(ndef['bgcolor']))
if cmndef == '':
continue
cmndef = cmndef.replace('$$', cp)
t2n[ttype] = name
c2d[name] = cmndef
def get_style_defs(self, arg=''):
"""
Return the command sequences needed to define the commands
used to format text in the verbatim environment. ``arg`` is ignored.
"""
cp = self.commandprefix
styles = []
for name, definition in iteritems(self.cmd2def):
styles.append(r'\expandafter\def\csname %s@tok@%s\endcsname{%s}' %
(cp, name, definition))
return STYLE_TEMPLATE % {'cp': self.commandprefix,
'styles': '\n'.join(styles)}
def format_unencoded(self, tokensource, outfile):
# TODO: add support for background colors
t2n = self.ttype2name
cp = self.commandprefix
if self.full:
realoutfile = outfile
outfile = StringIO()
outfile.write(u'\\begin{' + self.envname + u'}[commandchars=\\\\\\{\\}')
if self.linenos:
start, step = self.linenostart, self.linenostep
outfile.write(u',numbers=left' +
(start and u',firstnumber=%d' % start or u'') +
(step and u',stepnumber=%d' % step or u''))
if self.mathescape or self.texcomments or self.escapeinside:
outfile.write(u',codes={\\catcode`\\$=3\\catcode`\\^=7\\catcode`\\_=8}')
if self.verboptions:
outfile.write(u',' + self.verboptions)
outfile.write(u']\n')
for ttype, value in tokensource:
if ttype in Token.Comment:
if self.texcomments:
# Try to guess comment starting lexeme and escape it ...
start = value[0:1]
for i in xrange(1, len(value)):
if start[0] != value[i]:
break
start += value[i]
value = value[len(start):]
start = escape_tex(start, cp)
# ... but do not escape inside comment.
value = start + value
elif self.mathescape:
# Only escape parts not inside a math environment.
parts = value.split('$')
in_math = False
for i, part in enumerate(parts):
if not in_math:
parts[i] = escape_tex(part, cp)
in_math = not in_math
value = '$'.join(parts)
elif self.escapeinside:
text = value
value = ''
while text:
a, sep1, text = text.partition(self.left)
if sep1:
b, sep2, text = text.partition(self.right)
if sep2:
value += escape_tex(a, cp) + b
else:
value += escape_tex(a + sep1 + b, cp)
else:
value += escape_tex(a, cp)
else:
value = escape_tex(value, cp)
elif ttype not in Token.Escape:
value = escape_tex(value, cp)
styles = []
while ttype is not Token:
try:
styles.append(t2n[ttype])
except KeyError:
# not in current style
styles.append(_get_ttype_name(ttype))
ttype = ttype.parent
styleval = '+'.join(reversed(styles))
if styleval:
spl = value.split('\n')
for line in spl[:-1]:
if line:
outfile.write("\\%s{%s}{%s}" % (cp, styleval, line))
outfile.write('\n')
if spl[-1]:
outfile.write("\\%s{%s}{%s}" % (cp, styleval, spl[-1]))
else:
outfile.write(value)
outfile.write(u'\\end{' + self.envname + u'}\n')
if self.full:
encoding = self.encoding or 'utf8'
# map known existings encodings from LaTeX distribution
encoding = {
'utf_8': 'utf8',
'latin_1': 'latin1',
'iso_8859_1': 'latin1',
}.get(encoding.replace('-', '_'), encoding)
realoutfile.write(DOC_TEMPLATE %
dict(docclass = self.docclass,
preamble = self.preamble,
title = self.title,
encoding = encoding,
styledefs = self.get_style_defs(),
code = outfile.getvalue()))
class LatexEmbeddedLexer(Lexer):
"""
This lexer takes one lexer as argument, the lexer for the language
being formatted, and the left and right delimiters for escaped text.
First everything is scanned using the language lexer to obtain
strings and comments. All other consecutive tokens are merged and
the resulting text is scanned for escaped segments, which are given
the Token.Escape type. Finally text that is not escaped is scanned
again with the language lexer.
"""
def __init__(self, left, right, lang, **options):
self.left = left
self.right = right
self.lang = lang
Lexer.__init__(self, **options)
def get_tokens_unprocessed(self, text):
buf = ''
idx = 0
for i, t, v in self.lang.get_tokens_unprocessed(text):
if t in Token.Comment or t in Token.String:
if buf:
for x in self.get_tokens_aux(idx, buf):
yield x
buf = ''
yield i, t, v
else:
if not buf:
idx = i
buf += v
if buf:
for x in self.get_tokens_aux(idx, buf):
yield x
def get_tokens_aux(self, index, text):
while text:
a, sep1, text = text.partition(self.left)
if a:
for i, t, v in self.lang.get_tokens_unprocessed(a):
yield index + i, t, v
index += len(a)
if sep1:
b, sep2, text = text.partition(self.right)
if sep2:
yield index + len(sep1), Token.Escape, b
index += len(sep1) + len(b) + len(sep2)
else:
yield index, Token.Error, sep1
index += len(sep1)
text = b

View File

@ -1,160 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.formatters.other
~~~~~~~~~~~~~~~~~~~~~~~~~
Other formatters: NullFormatter, RawTokenFormatter.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from pygments.formatter import Formatter
from pygments.util import OptionError, get_choice_opt
from pygments.token import Token
from pygments.console import colorize
__all__ = ['NullFormatter', 'RawTokenFormatter', 'TestcaseFormatter']
class NullFormatter(Formatter):
"""
Output the text unchanged without any formatting.
"""
name = 'Text only'
aliases = ['text', 'null']
filenames = ['*.txt']
def format(self, tokensource, outfile):
enc = self.encoding
for ttype, value in tokensource:
if enc:
outfile.write(value.encode(enc))
else:
outfile.write(value)
class RawTokenFormatter(Formatter):
r"""
Format tokens as a raw representation for storing token streams.
The format is ``tokentype<TAB>repr(tokenstring)\n``. The output can later
be converted to a token stream with the `RawTokenLexer`, described in the
:doc:`lexer list <lexers>`.
Only two options are accepted:
`compress`
If set to ``'gz'`` or ``'bz2'``, compress the output with the given
compression algorithm after encoding (default: ``''``).
`error_color`
If set to a color name, highlight error tokens using that color. If
set but with no value, defaults to ``'red'``.
.. versionadded:: 0.11
"""
name = 'Raw tokens'
aliases = ['raw', 'tokens']
filenames = ['*.raw']
unicodeoutput = False
def __init__(self, **options):
Formatter.__init__(self, **options)
# We ignore self.encoding if it is set, since it gets set for lexer
# and formatter if given with -Oencoding on the command line.
# The RawTokenFormatter outputs only ASCII. Override here.
self.encoding = 'ascii' # let pygments.format() do the right thing
self.compress = get_choice_opt(options, 'compress',
['', 'none', 'gz', 'bz2'], '')
self.error_color = options.get('error_color', None)
if self.error_color is True:
self.error_color = 'red'
if self.error_color is not None:
try:
colorize(self.error_color, '')
except KeyError:
raise ValueError("Invalid color %r specified" %
self.error_color)
def format(self, tokensource, outfile):
try:
outfile.write(b'')
except TypeError:
raise TypeError('The raw tokens formatter needs a binary '
'output file')
if self.compress == 'gz':
import gzip
outfile = gzip.GzipFile('', 'wb', 9, outfile)
def write(text):
outfile.write(text.encode())
flush = outfile.flush
elif self.compress == 'bz2':
import bz2
compressor = bz2.BZ2Compressor(9)
def write(text):
outfile.write(compressor.compress(text.encode()))
def flush():
outfile.write(compressor.flush())
outfile.flush()
else:
def write(text):
outfile.write(text.encode())
flush = outfile.flush
if self.error_color:
for ttype, value in tokensource:
line = "%s\t%r\n" % (ttype, value)
if ttype is Token.Error:
write(colorize(self.error_color, line))
else:
write(line)
else:
for ttype, value in tokensource:
write("%s\t%r\n" % (ttype, value))
flush()
TESTCASE_BEFORE = u'''\
def testNeedsName(self):
fragment = %r
tokens = [
'''
TESTCASE_AFTER = u'''\
]
self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
'''
class TestcaseFormatter(Formatter):
"""
Format tokens as appropriate for a new testcase.
.. versionadded:: 2.0
"""
name = 'Testcase'
aliases = ['testcase']
def __init__(self, **options):
Formatter.__init__(self, **options)
if self.encoding is not None and self.encoding != 'utf-8':
raise ValueError("Only None and utf-8 are allowed encodings.")
def format(self, tokensource, outfile):
indentation = ' ' * 12
rawbuf = []
outbuf = []
for ttype, value in tokensource:
rawbuf.append(value)
outbuf.append('%s(%s, %r),\n' % (indentation, ttype, value))
before = TESTCASE_BEFORE % (u''.join(rawbuf),)
during = u''.join(outbuf)
after = TESTCASE_AFTER
if self.encoding is None:
outfile.write(before + during + after)
else:
outfile.write(before.encode('utf-8'))
outfile.write(during.encode('utf-8'))
outfile.write(after.encode('utf-8'))
outfile.flush()

View File

@ -1,147 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.formatters.rtf
~~~~~~~~~~~~~~~~~~~~~~~
A formatter that generates RTF files.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from pygments.formatter import Formatter
from pygments.util import get_int_opt, _surrogatepair
__all__ = ['RtfFormatter']
class RtfFormatter(Formatter):
"""
Format tokens as RTF markup. This formatter automatically outputs full RTF
documents with color information and other useful stuff. Perfect for Copy and
Paste into Microsoft(R) Word(R) documents.
Please note that ``encoding`` and ``outencoding`` options are ignored.
The RTF format is ASCII natively, but handles unicode characters correctly
thanks to escape sequences.
.. versionadded:: 0.6
Additional options accepted:
`style`
The style to use, can be a string or a Style subclass (default:
``'default'``).
`fontface`
The used font famliy, for example ``Bitstream Vera Sans``. Defaults to
some generic font which is supposed to have fixed width.
`fontsize`
Size of the font used. Size is specified in half points. The
default is 24 half-points, giving a size 12 font.
.. versionadded:: 2.0
"""
name = 'RTF'
aliases = ['rtf']
filenames = ['*.rtf']
def __init__(self, **options):
r"""
Additional options accepted:
``fontface``
Name of the font used. Could for example be ``'Courier New'``
to further specify the default which is ``'\fmodern'``. The RTF
specification claims that ``\fmodern`` are "Fixed-pitch serif
and sans serif fonts". Hope every RTF implementation thinks
the same about modern...
"""
Formatter.__init__(self, **options)
self.fontface = options.get('fontface') or ''
self.fontsize = get_int_opt(options, 'fontsize', 0)
def _escape(self, text):
return text.replace(u'\\', u'\\\\') \
.replace(u'{', u'\\{') \
.replace(u'}', u'\\}')
def _escape_text(self, text):
# empty strings, should give a small performance improvment
if not text:
return u''
# escape text
text = self._escape(text)
buf = []
for c in text:
cn = ord(c)
if cn < (2**7):
# ASCII character
buf.append(str(c))
elif (2**7) <= cn < (2**16):
# single unicode escape sequence
buf.append(u'{\\u%d}' % cn)
elif (2**16) <= cn:
# RTF limits unicode to 16 bits.
# Force surrogate pairs
buf.append(u'{\\u%d}{\\u%d}' % _surrogatepair(cn))
return u''.join(buf).replace(u'\n', u'\\par\n')
def format_unencoded(self, tokensource, outfile):
# rtf 1.8 header
outfile.write(u'{\\rtf1\\ansi\\uc0\\deff0'
u'{\\fonttbl{\\f0\\fmodern\\fprq1\\fcharset0%s;}}'
u'{\\colortbl;' % (self.fontface and
u' ' + self._escape(self.fontface) or
u''))
# convert colors and save them in a mapping to access them later.
color_mapping = {}
offset = 1
for _, style in self.style:
for color in style['color'], style['bgcolor'], style['border']:
if color and color not in color_mapping:
color_mapping[color] = offset
outfile.write(u'\\red%d\\green%d\\blue%d;' % (
int(color[0:2], 16),
int(color[2:4], 16),
int(color[4:6], 16)
))
offset += 1
outfile.write(u'}\\f0 ')
if self.fontsize:
outfile.write(u'\\fs%d' % (self.fontsize))
# highlight stream
for ttype, value in tokensource:
while not self.style.styles_token(ttype) and ttype.parent:
ttype = ttype.parent
style = self.style.style_for_token(ttype)
buf = []
if style['bgcolor']:
buf.append(u'\\cb%d' % color_mapping[style['bgcolor']])
if style['color']:
buf.append(u'\\cf%d' % color_mapping[style['color']])
if style['bold']:
buf.append(u'\\b')
if style['italic']:
buf.append(u'\\i')
if style['underline']:
buf.append(u'\\ul')
if style['border']:
buf.append(u'\\chbrdr\\chcfpat%d' %
color_mapping[style['border']])
start = u''.join(buf)
if start:
outfile.write(u'{%s ' % start)
outfile.write(self._escape_text(value))
if start:
outfile.write(u'}')
outfile.write(u'}')

View File

@ -1,153 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.formatters.svg
~~~~~~~~~~~~~~~~~~~~~~~
Formatter for SVG output.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from pygments.formatter import Formatter
from pygments.util import get_bool_opt, get_int_opt
__all__ = ['SvgFormatter']
def escape_html(text):
"""Escape &, <, > as well as single and double quotes for HTML."""
return text.replace('&', '&amp;'). \
replace('<', '&lt;'). \
replace('>', '&gt;'). \
replace('"', '&quot;'). \
replace("'", '&#39;')
class2style = {}
class SvgFormatter(Formatter):
"""
Format tokens as an SVG graphics file. This formatter is still experimental.
Each line of code is a ``<text>`` element with explicit ``x`` and ``y``
coordinates containing ``<tspan>`` elements with the individual token styles.
By default, this formatter outputs a full SVG document including doctype
declaration and the ``<svg>`` root element.
.. versionadded:: 0.9
Additional options accepted:
`nowrap`
Don't wrap the SVG ``<text>`` elements in ``<svg><g>`` elements and
don't add a XML declaration and a doctype. If true, the `fontfamily`
and `fontsize` options are ignored. Defaults to ``False``.
`fontfamily`
The value to give the wrapping ``<g>`` element's ``font-family``
attribute, defaults to ``"monospace"``.
`fontsize`
The value to give the wrapping ``<g>`` element's ``font-size``
attribute, defaults to ``"14px"``.
`xoffset`
Starting offset in X direction, defaults to ``0``.
`yoffset`
Starting offset in Y direction, defaults to the font size if it is given
in pixels, or ``20`` else. (This is necessary since text coordinates
refer to the text baseline, not the top edge.)
`ystep`
Offset to add to the Y coordinate for each subsequent line. This should
roughly be the text size plus 5. It defaults to that value if the text
size is given in pixels, or ``25`` else.
`spacehack`
Convert spaces in the source to ``&#160;``, which are non-breaking
spaces. SVG provides the ``xml:space`` attribute to control how
whitespace inside tags is handled, in theory, the ``preserve`` value
could be used to keep all whitespace as-is. However, many current SVG
viewers don't obey that rule, so this option is provided as a workaround
and defaults to ``True``.
"""
name = 'SVG'
aliases = ['svg']
filenames = ['*.svg']
def __init__(self, **options):
Formatter.__init__(self, **options)
self.nowrap = get_bool_opt(options, 'nowrap', False)
self.fontfamily = options.get('fontfamily', 'monospace')
self.fontsize = options.get('fontsize', '14px')
self.xoffset = get_int_opt(options, 'xoffset', 0)
fs = self.fontsize.strip()
if fs.endswith('px'): fs = fs[:-2].strip()
try:
int_fs = int(fs)
except:
int_fs = 20
self.yoffset = get_int_opt(options, 'yoffset', int_fs)
self.ystep = get_int_opt(options, 'ystep', int_fs + 5)
self.spacehack = get_bool_opt(options, 'spacehack', True)
self._stylecache = {}
def format_unencoded(self, tokensource, outfile):
"""
Format ``tokensource``, an iterable of ``(tokentype, tokenstring)``
tuples and write it into ``outfile``.
For our implementation we put all lines in their own 'line group'.
"""
x = self.xoffset
y = self.yoffset
if not self.nowrap:
if self.encoding:
outfile.write('<?xml version="1.0" encoding="%s"?>\n' %
self.encoding)
else:
outfile.write('<?xml version="1.0"?>\n')
outfile.write('<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN" '
'"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/'
'svg10.dtd">\n')
outfile.write('<svg xmlns="http://www.w3.org/2000/svg">\n')
outfile.write('<g font-family="%s" font-size="%s">\n' %
(self.fontfamily, self.fontsize))
outfile.write('<text x="%s" y="%s" xml:space="preserve">' % (x, y))
for ttype, value in tokensource:
style = self._get_style(ttype)
tspan = style and '<tspan' + style + '>' or ''
tspanend = tspan and '</tspan>' or ''
value = escape_html(value)
if self.spacehack:
value = value.expandtabs().replace(' ', '&#160;')
parts = value.split('\n')
for part in parts[:-1]:
outfile.write(tspan + part + tspanend)
y += self.ystep
outfile.write('</text>\n<text x="%s" y="%s" '
'xml:space="preserve">' % (x, y))
outfile.write(tspan + parts[-1] + tspanend)
outfile.write('</text>')
if not self.nowrap:
outfile.write('</g></svg>\n')
def _get_style(self, tokentype):
if tokentype in self._stylecache:
return self._stylecache[tokentype]
otokentype = tokentype
while not self.style.styles_token(tokentype):
tokentype = tokentype.parent
value = self.style.style_for_token(tokentype)
result = ''
if value['color']:
result = ' fill="#' + value['color'] + '"'
if value['bold']:
result += ' font-weight="bold"'
if value['italic']:
result += ' font-style="italic"'
self._stylecache[otokentype] = result
return result

View File

@ -1,136 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.formatters.terminal
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Formatter for terminal output with ANSI sequences.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import sys
from pygments.formatter import Formatter
from pygments.token import Keyword, Name, Comment, String, Error, \
Number, Operator, Generic, Token, Whitespace
from pygments.console import ansiformat
from pygments.util import get_choice_opt
__all__ = ['TerminalFormatter']
#: Map token types to a tuple of color values for light and dark
#: backgrounds.
TERMINAL_COLORS = {
Token: ('', ''),
Whitespace: ('lightgray', 'darkgray'),
Comment: ('lightgray', 'darkgray'),
Comment.Preproc: ('teal', 'turquoise'),
Keyword: ('darkblue', 'blue'),
Keyword.Type: ('teal', 'turquoise'),
Operator.Word: ('purple', 'fuchsia'),
Name.Builtin: ('teal', 'turquoise'),
Name.Function: ('darkgreen', 'green'),
Name.Namespace: ('_teal_', '_turquoise_'),
Name.Class: ('_darkgreen_', '_green_'),
Name.Exception: ('teal', 'turquoise'),
Name.Decorator: ('darkgray', 'lightgray'),
Name.Variable: ('darkred', 'red'),
Name.Constant: ('darkred', 'red'),
Name.Attribute: ('teal', 'turquoise'),
Name.Tag: ('blue', 'blue'),
String: ('brown', 'brown'),
Number: ('darkblue', 'blue'),
Generic.Deleted: ('red', 'red'),
Generic.Inserted: ('darkgreen', 'green'),
Generic.Heading: ('**', '**'),
Generic.Subheading: ('*purple*', '*fuchsia*'),
Generic.Prompt: ('**', '**'),
Generic.Error: ('red', 'red'),
Error: ('_red_', '_red_'),
}
class TerminalFormatter(Formatter):
r"""
Format tokens with ANSI color sequences, for output in a text console.
Color sequences are terminated at newlines, so that paging the output
works correctly.
The `get_style_defs()` method doesn't do anything special since there is
no support for common styles.
Options accepted:
`bg`
Set to ``"light"`` or ``"dark"`` depending on the terminal's background
(default: ``"light"``).
`colorscheme`
A dictionary mapping token types to (lightbg, darkbg) color names or
``None`` (default: ``None`` = use builtin colorscheme).
`linenos`
Set to ``True`` to have line numbers on the terminal output as well
(default: ``False`` = no line numbers).
"""
name = 'Terminal'
aliases = ['terminal', 'console']
filenames = []
def __init__(self, **options):
Formatter.__init__(self, **options)
self.darkbg = get_choice_opt(options, 'bg',
['light', 'dark'], 'light') == 'dark'
self.colorscheme = options.get('colorscheme', None) or TERMINAL_COLORS
self.linenos = options.get('linenos', False)
self._lineno = 0
def format(self, tokensource, outfile):
# hack: if the output is a terminal and has an encoding set,
# use that to avoid unicode encode problems
if not self.encoding and hasattr(outfile, "encoding") and \
hasattr(outfile, "isatty") and outfile.isatty() and \
sys.version_info < (3,):
self.encoding = outfile.encoding
return Formatter.format(self, tokensource, outfile)
def _write_lineno(self, outfile):
self._lineno += 1
outfile.write("%s%04d: " % (self._lineno != 1 and '\n' or '', self._lineno))
def _get_color(self, ttype):
# self.colorscheme is a dict containing usually generic types, so we
# have to walk the tree of dots. The base Token type must be a key,
# even if it's empty string, as in the default above.
colors = self.colorscheme.get(ttype)
while colors is None:
ttype = ttype.parent
colors = self.colorscheme.get(ttype)
return colors[self.darkbg]
def format_unencoded(self, tokensource, outfile):
if self.linenos:
self._write_lineno(outfile)
for ttype, value in tokensource:
color = self._get_color(ttype)
for line in value.splitlines(True):
if color:
outfile.write(ansiformat(color, line.rstrip('\n')))
else:
outfile.write(line.rstrip('\n'))
if line.endswith('\n'):
if self.linenos:
self._write_lineno(outfile)
else:
outfile.write('\n')
if self.linenos:
outfile.write("\n")

View File

@ -1,309 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.formatters.terminal256
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Formatter for 256-color terminal output with ANSI sequences.
RGB-to-XTERM color conversion routines adapted from xterm256-conv
tool (http://frexx.de/xterm-256-notes/data/xterm256-conv2.tar.bz2)
by Wolfgang Frisch.
Formatter version 1.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
# TODO:
# - Options to map style's bold/underline/italic/border attributes
# to some ANSI attrbutes (something like 'italic=underline')
# - An option to output "style RGB to xterm RGB/index" conversion table
# - An option to indicate that we are running in "reverse background"
# xterm. This means that default colors are white-on-black, not
# black-on-while, so colors like "white background" need to be converted
# to "white background, black foreground", etc...
import sys
from pygments.formatter import Formatter
from pygments.console import codes
from pygments.style import ansicolors
__all__ = ['Terminal256Formatter', 'TerminalTrueColorFormatter']
class EscapeSequence:
def __init__(self, fg=None, bg=None, bold=False, underline=False):
self.fg = fg
self.bg = bg
self.bold = bold
self.underline = underline
def escape(self, attrs):
if len(attrs):
return "\x1b[" + ";".join(attrs) + "m"
return ""
def color_string(self):
attrs = []
if self.fg is not None:
if self.fg in ansicolors:
esc = codes[self.fg[5:]]
if ';01m' in esc:
self.bold = True
# extract fg color code.
attrs.append(esc[2:4])
else:
attrs.extend(("38", "5", "%i" % self.fg))
if self.bg is not None:
if self.bg in ansicolors:
esc = codes[self.bg[5:]]
# extract fg color code, add 10 for bg.
attrs.append(str(int(esc[2:4])+10))
else:
attrs.extend(("48", "5", "%i" % self.bg))
if self.bold:
attrs.append("01")
if self.underline:
attrs.append("04")
return self.escape(attrs)
def true_color_string(self):
attrs = []
if self.fg:
attrs.extend(("38", "2", str(self.fg[0]), str(self.fg[1]), str(self.fg[2])))
if self.bg:
attrs.extend(("48", "2", str(self.bg[0]), str(self.bg[1]), str(self.bg[2])))
if self.bold:
attrs.append("01")
if self.underline:
attrs.append("04")
return self.escape(attrs)
def reset_string(self):
attrs = []
if self.fg is not None:
attrs.append("39")
if self.bg is not None:
attrs.append("49")
if self.bold or self.underline:
attrs.append("00")
return self.escape(attrs)
class Terminal256Formatter(Formatter):
"""
Format tokens with ANSI color sequences, for output in a 256-color
terminal or console. Like in `TerminalFormatter` color sequences
are terminated at newlines, so that paging the output works correctly.
The formatter takes colors from a style defined by the `style` option
and converts them to nearest ANSI 256-color escape sequences. Bold and
underline attributes from the style are preserved (and displayed).
.. versionadded:: 0.9
.. versionchanged:: 2.2
If the used style defines foreground colors in the form ``#ansi*``, then
`Terminal256Formatter` will map these to non extended foreground color.
See :ref:`AnsiTerminalStyle` for more information.
Options accepted:
`style`
The style to use, can be a string or a Style subclass (default:
``'default'``).
"""
name = 'Terminal256'
aliases = ['terminal256', 'console256', '256']
filenames = []
def __init__(self, **options):
Formatter.__init__(self, **options)
self.xterm_colors = []
self.best_match = {}
self.style_string = {}
self.usebold = 'nobold' not in options
self.useunderline = 'nounderline' not in options
self._build_color_table() # build an RGB-to-256 color conversion table
self._setup_styles() # convert selected style's colors to term. colors
def _build_color_table(self):
# colors 0..15: 16 basic colors
self.xterm_colors.append((0x00, 0x00, 0x00)) # 0
self.xterm_colors.append((0xcd, 0x00, 0x00)) # 1
self.xterm_colors.append((0x00, 0xcd, 0x00)) # 2
self.xterm_colors.append((0xcd, 0xcd, 0x00)) # 3
self.xterm_colors.append((0x00, 0x00, 0xee)) # 4
self.xterm_colors.append((0xcd, 0x00, 0xcd)) # 5
self.xterm_colors.append((0x00, 0xcd, 0xcd)) # 6
self.xterm_colors.append((0xe5, 0xe5, 0xe5)) # 7
self.xterm_colors.append((0x7f, 0x7f, 0x7f)) # 8
self.xterm_colors.append((0xff, 0x00, 0x00)) # 9
self.xterm_colors.append((0x00, 0xff, 0x00)) # 10
self.xterm_colors.append((0xff, 0xff, 0x00)) # 11
self.xterm_colors.append((0x5c, 0x5c, 0xff)) # 12
self.xterm_colors.append((0xff, 0x00, 0xff)) # 13
self.xterm_colors.append((0x00, 0xff, 0xff)) # 14
self.xterm_colors.append((0xff, 0xff, 0xff)) # 15
# colors 16..232: the 6x6x6 color cube
valuerange = (0x00, 0x5f, 0x87, 0xaf, 0xd7, 0xff)
for i in range(217):
r = valuerange[(i // 36) % 6]
g = valuerange[(i // 6) % 6]
b = valuerange[i % 6]
self.xterm_colors.append((r, g, b))
# colors 233..253: grayscale
for i in range(1, 22):
v = 8 + i * 10
self.xterm_colors.append((v, v, v))
def _closest_color(self, r, g, b):
distance = 257*257*3 # "infinity" (>distance from #000000 to #ffffff)
match = 0
for i in range(0, 254):
values = self.xterm_colors[i]
rd = r - values[0]
gd = g - values[1]
bd = b - values[2]
d = rd*rd + gd*gd + bd*bd
if d < distance:
match = i
distance = d
return match
def _color_index(self, color):
index = self.best_match.get(color, None)
if color in ansicolors:
# strip the `#ansi` part and look up code
index = color
self.best_match[color] = index
if index is None:
try:
rgb = int(str(color), 16)
except ValueError:
rgb = 0
r = (rgb >> 16) & 0xff
g = (rgb >> 8) & 0xff
b = rgb & 0xff
index = self._closest_color(r, g, b)
self.best_match[color] = index
return index
def _setup_styles(self):
for ttype, ndef in self.style:
escape = EscapeSequence()
# get foreground from ansicolor if set
if ndef['ansicolor']:
escape.fg = self._color_index(ndef['ansicolor'])
elif ndef['color']:
escape.fg = self._color_index(ndef['color'])
if ndef['bgansicolor']:
escape.bg = self._color_index(ndef['bgansicolor'])
elif ndef['bgcolor']:
escape.bg = self._color_index(ndef['bgcolor'])
if self.usebold and ndef['bold']:
escape.bold = True
if self.useunderline and ndef['underline']:
escape.underline = True
self.style_string[str(ttype)] = (escape.color_string(),
escape.reset_string())
def format(self, tokensource, outfile):
# hack: if the output is a terminal and has an encoding set,
# use that to avoid unicode encode problems
if not self.encoding and hasattr(outfile, "encoding") and \
hasattr(outfile, "isatty") and outfile.isatty() and \
sys.version_info < (3,):
self.encoding = outfile.encoding
return Formatter.format(self, tokensource, outfile)
def format_unencoded(self, tokensource, outfile):
for ttype, value in tokensource:
not_found = True
while ttype and not_found:
try:
# outfile.write( "<" + str(ttype) + ">" )
on, off = self.style_string[str(ttype)]
# Like TerminalFormatter, add "reset colors" escape sequence
# on newline.
spl = value.split('\n')
for line in spl[:-1]:
if line:
outfile.write(on + line + off)
outfile.write('\n')
if spl[-1]:
outfile.write(on + spl[-1] + off)
not_found = False
# outfile.write( '#' + str(ttype) + '#' )
except KeyError:
# ottype = ttype
ttype = ttype[:-1]
# outfile.write( '!' + str(ottype) + '->' + str(ttype) + '!' )
if not_found:
outfile.write(value)
class TerminalTrueColorFormatter(Terminal256Formatter):
r"""
Format tokens with ANSI color sequences, for output in a true-color
terminal or console. Like in `TerminalFormatter` color sequences
are terminated at newlines, so that paging the output works correctly.
.. versionadded:: 2.1
Options accepted:
`style`
The style to use, can be a string or a Style subclass (default:
``'default'``).
"""
name = 'TerminalTrueColor'
aliases = ['terminal16m', 'console16m', '16m']
filenames = []
def _build_color_table(self):
pass
def _color_tuple(self, color):
try:
rgb = int(str(color), 16)
except ValueError:
return None
r = (rgb >> 16) & 0xff
g = (rgb >> 8) & 0xff
b = rgb & 0xff
return (r, g, b)
def _setup_styles(self):
for ttype, ndef in self.style:
escape = EscapeSequence()
if ndef['color']:
escape.fg = self._color_tuple(ndef['color'])
if ndef['bgcolor']:
escape.bg = self._color_tuple(ndef['bgcolor'])
if self.usebold and ndef['bold']:
escape.bold = True
if self.useunderline and ndef['underline']:
escape.underline = True
self.style_string[str(ttype)] = (escape.true_color_string(),
escape.reset_string())

View File

@ -1,871 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexer
~~~~~~~~~~~~~~
Base lexer classes.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from __future__ import print_function
import re
import sys
import time
from pygments.filter import apply_filters, Filter
from pygments.filters import get_filter_by_name
from pygments.token import Error, Text, Other, _TokenType
from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
make_analysator, text_type, add_metaclass, iteritems, Future, guess_decode
from pygments.regexopt import regex_opt
__all__ = ['Lexer', 'RegexLexer', 'ExtendedRegexLexer', 'DelegatingLexer',
'LexerContext', 'include', 'inherit', 'bygroups', 'using', 'this',
'default', 'words']
_encoding_map = [(b'\xef\xbb\xbf', 'utf-8'),
(b'\xff\xfe\0\0', 'utf-32'),
(b'\0\0\xfe\xff', 'utf-32be'),
(b'\xff\xfe', 'utf-16'),
(b'\xfe\xff', 'utf-16be')]
_default_analyse = staticmethod(lambda x: 0.0)
class LexerMeta(type):
"""
This metaclass automagically converts ``analyse_text`` methods into
static methods which always return float values.
"""
def __new__(mcs, name, bases, d):
if 'analyse_text' in d:
d['analyse_text'] = make_analysator(d['analyse_text'])
return type.__new__(mcs, name, bases, d)
@add_metaclass(LexerMeta)
class Lexer(object):
"""
Lexer for a specific language.
Basic options recognized:
``stripnl``
Strip leading and trailing newlines from the input (default: True).
``stripall``
Strip all leading and trailing whitespace from the input
(default: False).
``ensurenl``
Make sure that the input ends with a newline (default: True). This
is required for some lexers that consume input linewise.
.. versionadded:: 1.3
``tabsize``
If given and greater than 0, expand tabs in the input (default: 0).
``encoding``
If given, must be an encoding name. This encoding will be used to
convert the input string to Unicode, if it is not already a Unicode
string (default: ``'guess'``, which uses a simple UTF-8 / Locale /
Latin1 detection. Can also be ``'chardet'`` to use the chardet
library, if it is installed.
``inencoding``
Overrides the ``encoding`` if given.
"""
#: Name of the lexer
name = None
#: Shortcuts for the lexer
aliases = []
#: File name globs
filenames = []
#: Secondary file name globs
alias_filenames = []
#: MIME types
mimetypes = []
#: Priority, should multiple lexers match and no content is provided
priority = 0
def __init__(self, **options):
self.options = options
self.stripnl = get_bool_opt(options, 'stripnl', True)
self.stripall = get_bool_opt(options, 'stripall', False)
self.ensurenl = get_bool_opt(options, 'ensurenl', True)
self.tabsize = get_int_opt(options, 'tabsize', 0)
self.encoding = options.get('encoding', 'guess')
self.encoding = options.get('inencoding') or self.encoding
self.filters = []
for filter_ in get_list_opt(options, 'filters', ()):
self.add_filter(filter_)
def __repr__(self):
if self.options:
return '<pygments.lexers.%s with %r>' % (self.__class__.__name__,
self.options)
else:
return '<pygments.lexers.%s>' % self.__class__.__name__
def add_filter(self, filter_, **options):
"""
Add a new stream filter to this lexer.
"""
if not isinstance(filter_, Filter):
filter_ = get_filter_by_name(filter_, **options)
self.filters.append(filter_)
def analyse_text(text):
"""
Has to return a float between ``0`` and ``1`` that indicates
if a lexer wants to highlight this text. Used by ``guess_lexer``.
If this method returns ``0`` it won't highlight it in any case, if
it returns ``1`` highlighting with this lexer is guaranteed.
The `LexerMeta` metaclass automatically wraps this function so
that it works like a static method (no ``self`` or ``cls``
parameter) and the return value is automatically converted to
`float`. If the return value is an object that is boolean `False`
it's the same as if the return values was ``0.0``.
"""
def get_tokens(self, text, unfiltered=False):
"""
Return an iterable of (tokentype, value) pairs generated from
`text`. If `unfiltered` is set to `True`, the filtering mechanism
is bypassed even if filters are defined.
Also preprocess the text, i.e. expand tabs and strip it if
wanted and applies registered filters.
"""
if not isinstance(text, text_type):
if self.encoding == 'guess':
text, _ = guess_decode(text)
elif self.encoding == 'chardet':
try:
import chardet
except ImportError:
raise ImportError('To enable chardet encoding guessing, '
'please install the chardet library '
'from http://chardet.feedparser.org/')
# check for BOM first
decoded = None
for bom, encoding in _encoding_map:
if text.startswith(bom):
decoded = text[len(bom):].decode(encoding, 'replace')
break
# no BOM found, so use chardet
if decoded is None:
enc = chardet.detect(text[:1024]) # Guess using first 1KB
decoded = text.decode(enc.get('encoding') or 'utf-8',
'replace')
text = decoded
else:
text = text.decode(self.encoding)
if text.startswith(u'\ufeff'):
text = text[len(u'\ufeff'):]
else:
if text.startswith(u'\ufeff'):
text = text[len(u'\ufeff'):]
# text now *is* a unicode string
text = text.replace('\r\n', '\n')
text = text.replace('\r', '\n')
if self.stripall:
text = text.strip()
elif self.stripnl:
text = text.strip('\n')
if self.tabsize > 0:
text = text.expandtabs(self.tabsize)
if self.ensurenl and not text.endswith('\n'):
text += '\n'
def streamer():
for _, t, v in self.get_tokens_unprocessed(text):
yield t, v
stream = streamer()
if not unfiltered:
stream = apply_filters(stream, self.filters, self)
return stream
def get_tokens_unprocessed(self, text):
"""
Return an iterable of (index, tokentype, value) pairs where "index"
is the starting position of the token within the input text.
In subclasses, implement this method as a generator to
maximize effectiveness.
"""
raise NotImplementedError
class DelegatingLexer(Lexer):
"""
This lexer takes two lexer as arguments. A root lexer and
a language lexer. First everything is scanned using the language
lexer, afterwards all ``Other`` tokens are lexed using the root
lexer.
The lexers from the ``template`` lexer package use this base lexer.
"""
def __init__(self, _root_lexer, _language_lexer, _needle=Other, **options):
self.root_lexer = _root_lexer(**options)
self.language_lexer = _language_lexer(**options)
self.needle = _needle
Lexer.__init__(self, **options)
def get_tokens_unprocessed(self, text):
buffered = ''
insertions = []
lng_buffer = []
for i, t, v in self.language_lexer.get_tokens_unprocessed(text):
if t is self.needle:
if lng_buffer:
insertions.append((len(buffered), lng_buffer))
lng_buffer = []
buffered += v
else:
lng_buffer.append((i, t, v))
if lng_buffer:
insertions.append((len(buffered), lng_buffer))
return do_insertions(insertions,
self.root_lexer.get_tokens_unprocessed(buffered))
# ------------------------------------------------------------------------------
# RegexLexer and ExtendedRegexLexer
#
class include(str): # pylint: disable=invalid-name
"""
Indicates that a state should include rules from another state.
"""
pass
class _inherit(object):
"""
Indicates the a state should inherit from its superclass.
"""
def __repr__(self):
return 'inherit'
inherit = _inherit() # pylint: disable=invalid-name
class combined(tuple): # pylint: disable=invalid-name
"""
Indicates a state combined from multiple states.
"""
def __new__(cls, *args):
return tuple.__new__(cls, args)
def __init__(self, *args):
# tuple.__init__ doesn't do anything
pass
class _PseudoMatch(object):
"""
A pseudo match object constructed from a string.
"""
def __init__(self, start, text):
self._text = text
self._start = start
def start(self, arg=None):
return self._start
def end(self, arg=None):
return self._start + len(self._text)
def group(self, arg=None):
if arg:
raise IndexError('No such group')
return self._text
def groups(self):
return (self._text,)
def groupdict(self):
return {}
def bygroups(*args):
"""
Callback that yields multiple actions for each group in the match.
"""
def callback(lexer, match, ctx=None):
for i, action in enumerate(args):
if action is None:
continue
elif type(action) is _TokenType:
data = match.group(i + 1)
if data:
yield match.start(i + 1), action, data
else:
data = match.group(i + 1)
if data is not None:
if ctx:
ctx.pos = match.start(i + 1)
for item in action(lexer,
_PseudoMatch(match.start(i + 1), data), ctx):
if item:
yield item
if ctx:
ctx.pos = match.end()
return callback
class _This(object):
"""
Special singleton used for indicating the caller class.
Used by ``using``.
"""
this = _This()
def using(_other, **kwargs):
"""
Callback that processes the match with a different lexer.
The keyword arguments are forwarded to the lexer, except `state` which
is handled separately.
`state` specifies the state that the new lexer will start in, and can
be an enumerable such as ('root', 'inline', 'string') or a simple
string which is assumed to be on top of the root state.
Note: For that to work, `_other` must not be an `ExtendedRegexLexer`.
"""
gt_kwargs = {}
if 'state' in kwargs:
s = kwargs.pop('state')
if isinstance(s, (list, tuple)):
gt_kwargs['stack'] = s
else:
gt_kwargs['stack'] = ('root', s)
if _other is this:
def callback(lexer, match, ctx=None):
# if keyword arguments are given the callback
# function has to create a new lexer instance
if kwargs:
# XXX: cache that somehow
kwargs.update(lexer.options)
lx = lexer.__class__(**kwargs)
else:
lx = lexer
s = match.start()
for i, t, v in lx.get_tokens_unprocessed(match.group(), **gt_kwargs):
yield i + s, t, v
if ctx:
ctx.pos = match.end()
else:
def callback(lexer, match, ctx=None):
# XXX: cache that somehow
kwargs.update(lexer.options)
lx = _other(**kwargs)
s = match.start()
for i, t, v in lx.get_tokens_unprocessed(match.group(), **gt_kwargs):
yield i + s, t, v
if ctx:
ctx.pos = match.end()
return callback
class default:
"""
Indicates a state or state action (e.g. #pop) to apply.
For example default('#pop') is equivalent to ('', Token, '#pop')
Note that state tuples may be used as well.
.. versionadded:: 2.0
"""
def __init__(self, state):
self.state = state
class words(Future):
"""
Indicates a list of literal words that is transformed into an optimized
regex that matches any of the words.
.. versionadded:: 2.0
"""
def __init__(self, words, prefix='', suffix=''):
self.words = words
self.prefix = prefix
self.suffix = suffix
def get(self):
return regex_opt(self.words, prefix=self.prefix, suffix=self.suffix)
class RegexLexerMeta(LexerMeta):
"""
Metaclass for RegexLexer, creates the self._tokens attribute from
self.tokens on the first instantiation.
"""
def _process_regex(cls, regex, rflags, state):
"""Preprocess the regular expression component of a token definition."""
if isinstance(regex, Future):
regex = regex.get()
return re.compile(regex, rflags).match
def _process_token(cls, token):
"""Preprocess the token component of a token definition."""
assert type(token) is _TokenType or callable(token), \
'token type must be simple type or callable, not %r' % (token,)
return token
def _process_new_state(cls, new_state, unprocessed, processed):
"""Preprocess the state transition action of a token definition."""
if isinstance(new_state, str):
# an existing state
if new_state == '#pop':
return -1
elif new_state in unprocessed:
return (new_state,)
elif new_state == '#push':
return new_state
elif new_state[:5] == '#pop:':
return -int(new_state[5:])
else:
assert False, 'unknown new state %r' % new_state
elif isinstance(new_state, combined):
# combine a new state from existing ones
tmp_state = '_tmp_%d' % cls._tmpname
cls._tmpname += 1
itokens = []
for istate in new_state:
assert istate != new_state, 'circular state ref %r' % istate
itokens.extend(cls._process_state(unprocessed,
processed, istate))
processed[tmp_state] = itokens
return (tmp_state,)
elif isinstance(new_state, tuple):
# push more than one state
for istate in new_state:
assert (istate in unprocessed or
istate in ('#pop', '#push')), \
'unknown new state ' + istate
return new_state
else:
assert False, 'unknown new state def %r' % new_state
def _process_state(cls, unprocessed, processed, state):
"""Preprocess a single state definition."""
assert type(state) is str, "wrong state name %r" % state
assert state[0] != '#', "invalid state name %r" % state
if state in processed:
return processed[state]
tokens = processed[state] = []
rflags = cls.flags
for tdef in unprocessed[state]:
if isinstance(tdef, include):
# it's a state reference
assert tdef != state, "circular state reference %r" % state
tokens.extend(cls._process_state(unprocessed, processed,
str(tdef)))
continue
if isinstance(tdef, _inherit):
# should be processed already, but may not in the case of:
# 1. the state has no counterpart in any parent
# 2. the state includes more than one 'inherit'
continue
if isinstance(tdef, default):
new_state = cls._process_new_state(tdef.state, unprocessed, processed)
tokens.append((re.compile('').match, None, new_state))
continue
assert type(tdef) is tuple, "wrong rule def %r" % tdef
try:
rex = cls._process_regex(tdef[0], rflags, state)
except Exception as err:
raise ValueError("uncompilable regex %r in state %r of %r: %s" %
(tdef[0], state, cls, err))
token = cls._process_token(tdef[1])
if len(tdef) == 2:
new_state = None
else:
new_state = cls._process_new_state(tdef[2],
unprocessed, processed)
tokens.append((rex, token, new_state))
return tokens
def process_tokendef(cls, name, tokendefs=None):
"""Preprocess a dictionary of token definitions."""
processed = cls._all_tokens[name] = {}
tokendefs = tokendefs or cls.tokens[name]
for state in list(tokendefs):
cls._process_state(tokendefs, processed, state)
return processed
def get_tokendefs(cls):
"""
Merge tokens from superclasses in MRO order, returning a single tokendef
dictionary.
Any state that is not defined by a subclass will be inherited
automatically. States that *are* defined by subclasses will, by
default, override that state in the superclass. If a subclass wishes to
inherit definitions from a superclass, it can use the special value
"inherit", which will cause the superclass' state definition to be
included at that point in the state.
"""
tokens = {}
inheritable = {}
for c in cls.__mro__:
toks = c.__dict__.get('tokens', {})
for state, items in iteritems(toks):
curitems = tokens.get(state)
if curitems is None:
# N.b. because this is assigned by reference, sufficiently
# deep hierarchies are processed incrementally (e.g. for
# A(B), B(C), C(RegexLexer), B will be premodified so X(B)
# will not see any inherits in B).
tokens[state] = items
try:
inherit_ndx = items.index(inherit)
except ValueError:
continue
inheritable[state] = inherit_ndx
continue
inherit_ndx = inheritable.pop(state, None)
if inherit_ndx is None:
continue
# Replace the "inherit" value with the items
curitems[inherit_ndx:inherit_ndx+1] = items
try:
# N.b. this is the index in items (that is, the superclass
# copy), so offset required when storing below.
new_inh_ndx = items.index(inherit)
except ValueError:
pass
else:
inheritable[state] = inherit_ndx + new_inh_ndx
return tokens
def __call__(cls, *args, **kwds):
"""Instantiate cls after preprocessing its token definitions."""
if '_tokens' not in cls.__dict__:
cls._all_tokens = {}
cls._tmpname = 0
if hasattr(cls, 'token_variants') and cls.token_variants:
# don't process yet
pass
else:
cls._tokens = cls.process_tokendef('', cls.get_tokendefs())
return type.__call__(cls, *args, **kwds)
@add_metaclass(RegexLexerMeta)
class RegexLexer(Lexer):
"""
Base for simple stateful regular expression-based lexers.
Simplifies the lexing process so that you need only
provide a list of states and regular expressions.
"""
#: Flags for compiling the regular expressions.
#: Defaults to MULTILINE.
flags = re.MULTILINE
#: Dict of ``{'state': [(regex, tokentype, new_state), ...], ...}``
#:
#: The initial state is 'root'.
#: ``new_state`` can be omitted to signify no state transition.
#: If it is a string, the state is pushed on the stack and changed.
#: If it is a tuple of strings, all states are pushed on the stack and
#: the current state will be the topmost.
#: It can also be ``combined('state1', 'state2', ...)``
#: to signify a new, anonymous state combined from the rules of two
#: or more existing ones.
#: Furthermore, it can be '#pop' to signify going back one step in
#: the state stack, or '#push' to push the current state on the stack
#: again.
#:
#: The tuple can also be replaced with ``include('state')``, in which
#: case the rules from the state named by the string are included in the
#: current one.
tokens = {}
def get_tokens_unprocessed(self, text, stack=('root',)):
"""
Split ``text`` into (tokentype, text) pairs.
``stack`` is the inital stack (default: ``['root']``)
"""
pos = 0
tokendefs = self._tokens
statestack = list(stack)
statetokens = tokendefs[statestack[-1]]
while 1:
for rexmatch, action, new_state in statetokens:
m = rexmatch(text, pos)
if m:
if action is not None:
if type(action) is _TokenType:
yield pos, action, m.group()
else:
for item in action(self, m):
yield item
pos = m.end()
if new_state is not None:
# state transition
if isinstance(new_state, tuple):
for state in new_state:
if state == '#pop':
statestack.pop()
elif state == '#push':
statestack.append(statestack[-1])
else:
statestack.append(state)
elif isinstance(new_state, int):
# pop
del statestack[new_state:]
elif new_state == '#push':
statestack.append(statestack[-1])
else:
assert False, "wrong state def: %r" % new_state
statetokens = tokendefs[statestack[-1]]
break
else:
# We are here only if all state tokens have been considered
# and there was not a match on any of them.
try:
if text[pos] == '\n':
# at EOL, reset state to "root"
statestack = ['root']
statetokens = tokendefs['root']
yield pos, Text, u'\n'
pos += 1
continue
yield pos, Error, text[pos]
pos += 1
except IndexError:
break
class LexerContext(object):
"""
A helper object that holds lexer position data.
"""
def __init__(self, text, pos, stack=None, end=None):
self.text = text
self.pos = pos
self.end = end or len(text) # end=0 not supported ;-)
self.stack = stack or ['root']
def __repr__(self):
return 'LexerContext(%r, %r, %r)' % (
self.text, self.pos, self.stack)
class ExtendedRegexLexer(RegexLexer):
"""
A RegexLexer that uses a context object to store its state.
"""
def get_tokens_unprocessed(self, text=None, context=None):
"""
Split ``text`` into (tokentype, text) pairs.
If ``context`` is given, use this lexer context instead.
"""
tokendefs = self._tokens
if not context:
ctx = LexerContext(text, 0)
statetokens = tokendefs['root']
else:
ctx = context
statetokens = tokendefs[ctx.stack[-1]]
text = ctx.text
while 1:
for rexmatch, action, new_state in statetokens:
m = rexmatch(text, ctx.pos, ctx.end)
if m:
if action is not None:
if type(action) is _TokenType:
yield ctx.pos, action, m.group()
ctx.pos = m.end()
else:
for item in action(self, m, ctx):
yield item
if not new_state:
# altered the state stack?
statetokens = tokendefs[ctx.stack[-1]]
# CAUTION: callback must set ctx.pos!
if new_state is not None:
# state transition
if isinstance(new_state, tuple):
for state in new_state:
if state == '#pop':
ctx.stack.pop()
elif state == '#push':
ctx.stack.append(ctx.stack[-1])
else:
ctx.stack.append(state)
elif isinstance(new_state, int):
# pop
del ctx.stack[new_state:]
elif new_state == '#push':
ctx.stack.append(ctx.stack[-1])
else:
assert False, "wrong state def: %r" % new_state
statetokens = tokendefs[ctx.stack[-1]]
break
else:
try:
if ctx.pos >= ctx.end:
break
if text[ctx.pos] == '\n':
# at EOL, reset state to "root"
ctx.stack = ['root']
statetokens = tokendefs['root']
yield ctx.pos, Text, u'\n'
ctx.pos += 1
continue
yield ctx.pos, Error, text[ctx.pos]
ctx.pos += 1
except IndexError:
break
def do_insertions(insertions, tokens):
"""
Helper for lexers which must combine the results of several
sublexers.
``insertions`` is a list of ``(index, itokens)`` pairs.
Each ``itokens`` iterable should be inserted at position
``index`` into the token stream given by the ``tokens``
argument.
The result is a combined token stream.
TODO: clean up the code here.
"""
insertions = iter(insertions)
try:
index, itokens = next(insertions)
except StopIteration:
# no insertions
for item in tokens:
yield item
return
realpos = None
insleft = True
# iterate over the token stream where we want to insert
# the tokens from the insertion list.
for i, t, v in tokens:
# first iteration. store the postition of first item
if realpos is None:
realpos = i
oldi = 0
while insleft and i + len(v) >= index:
tmpval = v[oldi:index - i]
yield realpos, t, tmpval
realpos += len(tmpval)
for it_index, it_token, it_value in itokens:
yield realpos, it_token, it_value
realpos += len(it_value)
oldi = index - i
try:
index, itokens = next(insertions)
except StopIteration:
insleft = False
break # not strictly necessary
yield realpos, t, v[oldi:]
realpos += len(v) - oldi
# leftover tokens
while insleft:
# no normal tokens, set realpos to zero
realpos = realpos or 0
for p, t, v in itokens:
yield realpos, t, v
realpos += len(v)
try:
index, itokens = next(insertions)
except StopIteration:
insleft = False
break # not strictly necessary
class ProfilingRegexLexerMeta(RegexLexerMeta):
"""Metaclass for ProfilingRegexLexer, collects regex timing info."""
def _process_regex(cls, regex, rflags, state):
if isinstance(regex, words):
rex = regex_opt(regex.words, prefix=regex.prefix,
suffix=regex.suffix)
else:
rex = regex
compiled = re.compile(rex, rflags)
def match_func(text, pos, endpos=sys.maxsize):
info = cls._prof_data[-1].setdefault((state, rex), [0, 0.0])
t0 = time.time()
res = compiled.match(text, pos, endpos)
t1 = time.time()
info[0] += 1
info[1] += t1 - t0
return res
return match_func
@add_metaclass(ProfilingRegexLexerMeta)
class ProfilingRegexLexer(RegexLexer):
"""Drop-in replacement for RegexLexer that does profiling of its regexes."""
_prof_data = []
_prof_sort_index = 4 # defaults to time per call
def get_tokens_unprocessed(self, text, stack=('root',)):
# this needs to be a stack, since using(this) will produce nested calls
self.__class__._prof_data.append({})
for tok in RegexLexer.get_tokens_unprocessed(self, text, stack):
yield tok
rawdata = self.__class__._prof_data.pop()
data = sorted(((s, repr(r).strip('u\'').replace('\\\\', '\\')[:65],
n, 1000 * t, 1000 * t / n)
for ((s, r), (n, t)) in rawdata.items()),
key=lambda x: x[self._prof_sort_index],
reverse=True)
sum_total = sum(x[3] for x in data)
print()
print('Profiling result for %s lexing %d chars in %.3f ms' %
(self.__class__.__name__, len(text), sum_total))
print('=' * 110)
print('%-20s %-64s ncalls tottime percall' % ('state', 'regex'))
print('-' * 110)
for d in data:
print('%-20s %-65s %5d %8.4f %8.4f' % d)
print('=' * 110)

View File

@ -1,329 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexers
~~~~~~~~~~~~~~~
Pygments lexers.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import re
import sys
import types
import fnmatch
from os.path import basename
from pygments.lexers._mapping import LEXERS
from pygments.modeline import get_filetype_from_buffer
from pygments.plugin import find_plugin_lexers
from pygments.util import ClassNotFound, itervalues, guess_decode
__all__ = ['get_lexer_by_name', 'get_lexer_for_filename', 'find_lexer_class',
'guess_lexer', 'load_lexer_from_file'] + list(LEXERS)
_lexer_cache = {}
_pattern_cache = {}
def _fn_matches(fn, glob):
"""Return whether the supplied file name fn matches pattern filename."""
if glob not in _pattern_cache:
pattern = _pattern_cache[glob] = re.compile(fnmatch.translate(glob))
return pattern.match(fn)
return _pattern_cache[glob].match(fn)
def _load_lexers(module_name):
"""Load a lexer (and all others in the module too)."""
mod = __import__(module_name, None, None, ['__all__'])
for lexer_name in mod.__all__:
cls = getattr(mod, lexer_name)
_lexer_cache[cls.name] = cls
def get_all_lexers():
"""Return a generator of tuples in the form ``(name, aliases,
filenames, mimetypes)`` of all know lexers.
"""
for item in itervalues(LEXERS):
yield item[1:]
for lexer in find_plugin_lexers():
yield lexer.name, lexer.aliases, lexer.filenames, lexer.mimetypes
def find_lexer_class(name):
"""Lookup a lexer class by name.
Return None if not found.
"""
if name in _lexer_cache:
return _lexer_cache[name]
# lookup builtin lexers
for module_name, lname, aliases, _, _ in itervalues(LEXERS):
if name == lname:
_load_lexers(module_name)
return _lexer_cache[name]
# continue with lexers from setuptools entrypoints
for cls in find_plugin_lexers():
if cls.name == name:
return cls
def find_lexer_class_by_name(_alias):
"""Lookup a lexer class by alias.
Like `get_lexer_by_name`, but does not instantiate the class.
.. versionadded:: 2.2
"""
if not _alias:
raise ClassNotFound('no lexer for alias %r found' % _alias)
# lookup builtin lexers
for module_name, name, aliases, _, _ in itervalues(LEXERS):
if _alias.lower() in aliases:
if name not in _lexer_cache:
_load_lexers(module_name)
return _lexer_cache[name]
# continue with lexers from setuptools entrypoints
for cls in find_plugin_lexers():
if _alias.lower() in cls.aliases:
return cls
raise ClassNotFound('no lexer for alias %r found' % _alias)
def get_lexer_by_name(_alias, **options):
"""Get a lexer by an alias.
Raises ClassNotFound if not found.
"""
if not _alias:
raise ClassNotFound('no lexer for alias %r found' % _alias)
# lookup builtin lexers
for module_name, name, aliases, _, _ in itervalues(LEXERS):
if _alias.lower() in aliases:
if name not in _lexer_cache:
_load_lexers(module_name)
return _lexer_cache[name](**options)
# continue with lexers from setuptools entrypoints
for cls in find_plugin_lexers():
if _alias.lower() in cls.aliases:
return cls(**options)
raise ClassNotFound('no lexer for alias %r found' % _alias)
def load_lexer_from_file(filename, lexername="CustomLexer", **options):
"""Load a lexer from a file.
This method expects a file located relative to the current working
directory, which contains a Lexer class. By default, it expects the
Lexer to be name CustomLexer; you can specify your own class name
as the second argument to this function.
Users should be very careful with the input, because this method
is equivalent to running eval on the input file.
Raises ClassNotFound if there are any problems importing the Lexer.
.. versionadded:: 2.2
"""
try:
# This empty dict will contain the namespace for the exec'd file
custom_namespace = {}
exec(open(filename, 'rb').read(), custom_namespace)
# Retrieve the class `lexername` from that namespace
if lexername not in custom_namespace:
raise ClassNotFound('no valid %s class found in %s' %
(lexername, filename))
lexer_class = custom_namespace[lexername]
# And finally instantiate it with the options
return lexer_class(**options)
except IOError as err:
raise ClassNotFound('cannot read %s' % filename)
except ClassNotFound as err:
raise
except Exception as err:
raise ClassNotFound('error when loading custom lexer: %s' % err)
def find_lexer_class_for_filename(_fn, code=None):
"""Get a lexer for a filename.
If multiple lexers match the filename pattern, use ``analyse_text()`` to
figure out which one is more appropriate.
Returns None if not found.
"""
matches = []
fn = basename(_fn)
for modname, name, _, filenames, _ in itervalues(LEXERS):
for filename in filenames:
if _fn_matches(fn, filename):
if name not in _lexer_cache:
_load_lexers(modname)
matches.append((_lexer_cache[name], filename))
for cls in find_plugin_lexers():
for filename in cls.filenames:
if _fn_matches(fn, filename):
matches.append((cls, filename))
if sys.version_info > (3,) and isinstance(code, bytes):
# decode it, since all analyse_text functions expect unicode
code = guess_decode(code)
def get_rating(info):
cls, filename = info
# explicit patterns get a bonus
bonus = '*' not in filename and 0.5 or 0
# The class _always_ defines analyse_text because it's included in
# the Lexer class. The default implementation returns None which
# gets turned into 0.0. Run scripts/detect_missing_analyse_text.py
# to find lexers which need it overridden.
if code:
return cls.analyse_text(code) + bonus, cls.__name__
return cls.priority + bonus, cls.__name__
if matches:
matches.sort(key=get_rating)
# print "Possible lexers, after sort:", matches
return matches[-1][0]
def get_lexer_for_filename(_fn, code=None, **options):
"""Get a lexer for a filename.
If multiple lexers match the filename pattern, use ``analyse_text()`` to
figure out which one is more appropriate.
Raises ClassNotFound if not found.
"""
res = find_lexer_class_for_filename(_fn, code)
if not res:
raise ClassNotFound('no lexer for filename %r found' % _fn)
return res(**options)
def get_lexer_for_mimetype(_mime, **options):
"""Get a lexer for a mimetype.
Raises ClassNotFound if not found.
"""
for modname, name, _, _, mimetypes in itervalues(LEXERS):
if _mime in mimetypes:
if name not in _lexer_cache:
_load_lexers(modname)
return _lexer_cache[name](**options)
for cls in find_plugin_lexers():
if _mime in cls.mimetypes:
return cls(**options)
raise ClassNotFound('no lexer for mimetype %r found' % _mime)
def _iter_lexerclasses(plugins=True):
"""Return an iterator over all lexer classes."""
for key in sorted(LEXERS):
module_name, name = LEXERS[key][:2]
if name not in _lexer_cache:
_load_lexers(module_name)
yield _lexer_cache[name]
if plugins:
for lexer in find_plugin_lexers():
yield lexer
def guess_lexer_for_filename(_fn, _text, **options):
"""
Lookup all lexers that handle those filenames primary (``filenames``)
or secondary (``alias_filenames``). Then run a text analysis for those
lexers and choose the best result.
usage::
>>> from pygments.lexers import guess_lexer_for_filename
>>> guess_lexer_for_filename('hello.html', '<%= @foo %>')
<pygments.lexers.templates.RhtmlLexer object at 0xb7d2f32c>
>>> guess_lexer_for_filename('hello.html', '<h1>{{ title|e }}</h1>')
<pygments.lexers.templates.HtmlDjangoLexer object at 0xb7d2f2ac>
>>> guess_lexer_for_filename('style.css', 'a { color: <?= $link ?> }')
<pygments.lexers.templates.CssPhpLexer object at 0xb7ba518c>
"""
fn = basename(_fn)
primary = {}
matching_lexers = set()
for lexer in _iter_lexerclasses():
for filename in lexer.filenames:
if _fn_matches(fn, filename):
matching_lexers.add(lexer)
primary[lexer] = True
for filename in lexer.alias_filenames:
if _fn_matches(fn, filename):
matching_lexers.add(lexer)
primary[lexer] = False
if not matching_lexers:
raise ClassNotFound('no lexer for filename %r found' % fn)
if len(matching_lexers) == 1:
return matching_lexers.pop()(**options)
result = []
for lexer in matching_lexers:
rv = lexer.analyse_text(_text)
if rv == 1.0:
return lexer(**options)
result.append((rv, lexer))
def type_sort(t):
# sort by:
# - analyse score
# - is primary filename pattern?
# - priority
# - last resort: class name
return (t[0], primary[t[1]], t[1].priority, t[1].__name__)
result.sort(key=type_sort)
return result[-1][1](**options)
def guess_lexer(_text, **options):
"""Guess a lexer by strong distinctions in the text (eg, shebang)."""
# try to get a vim modeline first
ft = get_filetype_from_buffer(_text)
if ft is not None:
try:
return get_lexer_by_name(ft, **options)
except ClassNotFound:
pass
best_lexer = [0.0, None]
for lexer in _iter_lexerclasses():
rv = lexer.analyse_text(_text)
if rv == 1.0:
return lexer(**options)
if rv > best_lexer[0]:
best_lexer[:] = (rv, lexer)
if not best_lexer[0] or best_lexer[1] is None:
raise ClassNotFound('no lexer matching the text found')
return best_lexer[1](**options)
class _automodule(types.ModuleType):
"""Automatically import lexers."""
def __getattr__(self, name):
info = LEXERS.get(name)
if info:
_load_lexers(info[0])
cls = _lexer_cache[info[1]]
setattr(self, name, cls)
return cls
raise AttributeError(name)
oldmod = sys.modules[__name__]
newmod = _automodule(__name__)
newmod.__dict__.update(oldmod.__dict__)
sys.modules[__name__] = newmod
del newmod.newmod, newmod.oldmod, newmod.sys, newmod.types

File diff suppressed because it is too large Load Diff

View File

@ -1,232 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexers._cl_builtins
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
ANSI Common Lisp builtins.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
BUILTIN_FUNCTIONS = set(( # 638 functions
'<', '<=', '=', '>', '>=', '-', '/', '/=', '*', '+', '1-', '1+',
'abort', 'abs', 'acons', 'acos', 'acosh', 'add-method', 'adjoin',
'adjustable-array-p', 'adjust-array', 'allocate-instance',
'alpha-char-p', 'alphanumericp', 'append', 'apply', 'apropos',
'apropos-list', 'aref', 'arithmetic-error-operands',
'arithmetic-error-operation', 'array-dimension', 'array-dimensions',
'array-displacement', 'array-element-type', 'array-has-fill-pointer-p',
'array-in-bounds-p', 'arrayp', 'array-rank', 'array-row-major-index',
'array-total-size', 'ash', 'asin', 'asinh', 'assoc', 'assoc-if',
'assoc-if-not', 'atan', 'atanh', 'atom', 'bit', 'bit-and', 'bit-andc1',
'bit-andc2', 'bit-eqv', 'bit-ior', 'bit-nand', 'bit-nor', 'bit-not',
'bit-orc1', 'bit-orc2', 'bit-vector-p', 'bit-xor', 'boole',
'both-case-p', 'boundp', 'break', 'broadcast-stream-streams',
'butlast', 'byte', 'byte-position', 'byte-size', 'caaaar', 'caaadr',
'caaar', 'caadar', 'caaddr', 'caadr', 'caar', 'cadaar', 'cadadr',
'cadar', 'caddar', 'cadddr', 'caddr', 'cadr', 'call-next-method', 'car',
'cdaaar', 'cdaadr', 'cdaar', 'cdadar', 'cdaddr', 'cdadr', 'cdar',
'cddaar', 'cddadr', 'cddar', 'cdddar', 'cddddr', 'cdddr', 'cddr', 'cdr',
'ceiling', 'cell-error-name', 'cerror', 'change-class', 'char', 'char<',
'char<=', 'char=', 'char>', 'char>=', 'char/=', 'character',
'characterp', 'char-code', 'char-downcase', 'char-equal',
'char-greaterp', 'char-int', 'char-lessp', 'char-name',
'char-not-equal', 'char-not-greaterp', 'char-not-lessp', 'char-upcase',
'cis', 'class-name', 'class-of', 'clear-input', 'clear-output',
'close', 'clrhash', 'code-char', 'coerce', 'compile',
'compiled-function-p', 'compile-file', 'compile-file-pathname',
'compiler-macro-function', 'complement', 'complex', 'complexp',
'compute-applicable-methods', 'compute-restarts', 'concatenate',
'concatenated-stream-streams', 'conjugate', 'cons', 'consp',
'constantly', 'constantp', 'continue', 'copy-alist', 'copy-list',
'copy-pprint-dispatch', 'copy-readtable', 'copy-seq', 'copy-structure',
'copy-symbol', 'copy-tree', 'cos', 'cosh', 'count', 'count-if',
'count-if-not', 'decode-float', 'decode-universal-time', 'delete',
'delete-duplicates', 'delete-file', 'delete-if', 'delete-if-not',
'delete-package', 'denominator', 'deposit-field', 'describe',
'describe-object', 'digit-char', 'digit-char-p', 'directory',
'directory-namestring', 'disassemble', 'documentation', 'dpb',
'dribble', 'echo-stream-input-stream', 'echo-stream-output-stream',
'ed', 'eighth', 'elt', 'encode-universal-time', 'endp',
'enough-namestring', 'ensure-directories-exist',
'ensure-generic-function', 'eq', 'eql', 'equal', 'equalp', 'error',
'eval', 'evenp', 'every', 'exp', 'export', 'expt', 'fboundp',
'fceiling', 'fdefinition', 'ffloor', 'fifth', 'file-author',
'file-error-pathname', 'file-length', 'file-namestring',
'file-position', 'file-string-length', 'file-write-date',
'fill', 'fill-pointer', 'find', 'find-all-symbols', 'find-class',
'find-if', 'find-if-not', 'find-method', 'find-package', 'find-restart',
'find-symbol', 'finish-output', 'first', 'float', 'float-digits',
'floatp', 'float-precision', 'float-radix', 'float-sign', 'floor',
'fmakunbound', 'force-output', 'format', 'fourth', 'fresh-line',
'fround', 'ftruncate', 'funcall', 'function-keywords',
'function-lambda-expression', 'functionp', 'gcd', 'gensym', 'gentemp',
'get', 'get-decoded-time', 'get-dispatch-macro-character', 'getf',
'gethash', 'get-internal-real-time', 'get-internal-run-time',
'get-macro-character', 'get-output-stream-string', 'get-properties',
'get-setf-expansion', 'get-universal-time', 'graphic-char-p',
'hash-table-count', 'hash-table-p', 'hash-table-rehash-size',
'hash-table-rehash-threshold', 'hash-table-size', 'hash-table-test',
'host-namestring', 'identity', 'imagpart', 'import',
'initialize-instance', 'input-stream-p', 'inspect',
'integer-decode-float', 'integer-length', 'integerp',
'interactive-stream-p', 'intern', 'intersection',
'invalid-method-error', 'invoke-debugger', 'invoke-restart',
'invoke-restart-interactively', 'isqrt', 'keywordp', 'last', 'lcm',
'ldb', 'ldb-test', 'ldiff', 'length', 'lisp-implementation-type',
'lisp-implementation-version', 'list', 'list*', 'list-all-packages',
'listen', 'list-length', 'listp', 'load',
'load-logical-pathname-translations', 'log', 'logand', 'logandc1',
'logandc2', 'logbitp', 'logcount', 'logeqv', 'logical-pathname',
'logical-pathname-translations', 'logior', 'lognand', 'lognor',
'lognot', 'logorc1', 'logorc2', 'logtest', 'logxor', 'long-site-name',
'lower-case-p', 'machine-instance', 'machine-type', 'machine-version',
'macroexpand', 'macroexpand-1', 'macro-function', 'make-array',
'make-broadcast-stream', 'make-concatenated-stream', 'make-condition',
'make-dispatch-macro-character', 'make-echo-stream', 'make-hash-table',
'make-instance', 'make-instances-obsolete', 'make-list',
'make-load-form', 'make-load-form-saving-slots', 'make-package',
'make-pathname', 'make-random-state', 'make-sequence', 'make-string',
'make-string-input-stream', 'make-string-output-stream', 'make-symbol',
'make-synonym-stream', 'make-two-way-stream', 'makunbound', 'map',
'mapc', 'mapcan', 'mapcar', 'mapcon', 'maphash', 'map-into', 'mapl',
'maplist', 'mask-field', 'max', 'member', 'member-if', 'member-if-not',
'merge', 'merge-pathnames', 'method-combination-error',
'method-qualifiers', 'min', 'minusp', 'mismatch', 'mod',
'muffle-warning', 'name-char', 'namestring', 'nbutlast', 'nconc',
'next-method-p', 'nintersection', 'ninth', 'no-applicable-method',
'no-next-method', 'not', 'notany', 'notevery', 'nreconc', 'nreverse',
'nset-difference', 'nset-exclusive-or', 'nstring-capitalize',
'nstring-downcase', 'nstring-upcase', 'nsublis', 'nsubst', 'nsubst-if',
'nsubst-if-not', 'nsubstitute', 'nsubstitute-if', 'nsubstitute-if-not',
'nth', 'nthcdr', 'null', 'numberp', 'numerator', 'nunion', 'oddp',
'open', 'open-stream-p', 'output-stream-p', 'package-error-package',
'package-name', 'package-nicknames', 'packagep',
'package-shadowing-symbols', 'package-used-by-list', 'package-use-list',
'pairlis', 'parse-integer', 'parse-namestring', 'pathname',
'pathname-device', 'pathname-directory', 'pathname-host',
'pathname-match-p', 'pathname-name', 'pathnamep', 'pathname-type',
'pathname-version', 'peek-char', 'phase', 'plusp', 'position',
'position-if', 'position-if-not', 'pprint', 'pprint-dispatch',
'pprint-fill', 'pprint-indent', 'pprint-linear', 'pprint-newline',
'pprint-tab', 'pprint-tabular', 'prin1', 'prin1-to-string', 'princ',
'princ-to-string', 'print', 'print-object', 'probe-file', 'proclaim',
'provide', 'random', 'random-state-p', 'rassoc', 'rassoc-if',
'rassoc-if-not', 'rational', 'rationalize', 'rationalp', 'read',
'read-byte', 'read-char', 'read-char-no-hang', 'read-delimited-list',
'read-from-string', 'read-line', 'read-preserving-whitespace',
'read-sequence', 'readtable-case', 'readtablep', 'realp', 'realpart',
'reduce', 'reinitialize-instance', 'rem', 'remhash', 'remove',
'remove-duplicates', 'remove-if', 'remove-if-not', 'remove-method',
'remprop', 'rename-file', 'rename-package', 'replace', 'require',
'rest', 'restart-name', 'revappend', 'reverse', 'room', 'round',
'row-major-aref', 'rplaca', 'rplacd', 'sbit', 'scale-float', 'schar',
'search', 'second', 'set', 'set-difference',
'set-dispatch-macro-character', 'set-exclusive-or',
'set-macro-character', 'set-pprint-dispatch', 'set-syntax-from-char',
'seventh', 'shadow', 'shadowing-import', 'shared-initialize',
'short-site-name', 'signal', 'signum', 'simple-bit-vector-p',
'simple-condition-format-arguments', 'simple-condition-format-control',
'simple-string-p', 'simple-vector-p', 'sin', 'sinh', 'sixth', 'sleep',
'slot-boundp', 'slot-exists-p', 'slot-makunbound', 'slot-missing',
'slot-unbound', 'slot-value', 'software-type', 'software-version',
'some', 'sort', 'special-operator-p', 'sqrt', 'stable-sort',
'standard-char-p', 'store-value', 'stream-element-type',
'stream-error-stream', 'stream-external-format', 'streamp', 'string',
'string<', 'string<=', 'string=', 'string>', 'string>=', 'string/=',
'string-capitalize', 'string-downcase', 'string-equal',
'string-greaterp', 'string-left-trim', 'string-lessp',
'string-not-equal', 'string-not-greaterp', 'string-not-lessp',
'stringp', 'string-right-trim', 'string-trim', 'string-upcase',
'sublis', 'subseq', 'subsetp', 'subst', 'subst-if', 'subst-if-not',
'substitute', 'substitute-if', 'substitute-if-not', 'subtypep','svref',
'sxhash', 'symbol-function', 'symbol-name', 'symbolp', 'symbol-package',
'symbol-plist', 'symbol-value', 'synonym-stream-symbol', 'syntax:',
'tailp', 'tan', 'tanh', 'tenth', 'terpri', 'third',
'translate-logical-pathname', 'translate-pathname', 'tree-equal',
'truename', 'truncate', 'two-way-stream-input-stream',
'two-way-stream-output-stream', 'type-error-datum',
'type-error-expected-type', 'type-of', 'typep', 'unbound-slot-instance',
'unexport', 'unintern', 'union', 'unread-char', 'unuse-package',
'update-instance-for-different-class',
'update-instance-for-redefined-class', 'upgraded-array-element-type',
'upgraded-complex-part-type', 'upper-case-p', 'use-package',
'user-homedir-pathname', 'use-value', 'values', 'values-list', 'vector',
'vectorp', 'vector-pop', 'vector-push', 'vector-push-extend', 'warn',
'wild-pathname-p', 'write', 'write-byte', 'write-char', 'write-line',
'write-sequence', 'write-string', 'write-to-string', 'yes-or-no-p',
'y-or-n-p', 'zerop',
))
SPECIAL_FORMS = set((
'block', 'catch', 'declare', 'eval-when', 'flet', 'function', 'go', 'if',
'labels', 'lambda', 'let', 'let*', 'load-time-value', 'locally', 'macrolet',
'multiple-value-call', 'multiple-value-prog1', 'progn', 'progv', 'quote',
'return-from', 'setq', 'symbol-macrolet', 'tagbody', 'the', 'throw',
'unwind-protect',
))
MACROS = set((
'and', 'assert', 'call-method', 'case', 'ccase', 'check-type', 'cond',
'ctypecase', 'decf', 'declaim', 'defclass', 'defconstant', 'defgeneric',
'define-compiler-macro', 'define-condition', 'define-method-combination',
'define-modify-macro', 'define-setf-expander', 'define-symbol-macro',
'defmacro', 'defmethod', 'defpackage', 'defparameter', 'defsetf',
'defstruct', 'deftype', 'defun', 'defvar', 'destructuring-bind', 'do',
'do*', 'do-all-symbols', 'do-external-symbols', 'dolist', 'do-symbols',
'dotimes', 'ecase', 'etypecase', 'formatter', 'handler-bind',
'handler-case', 'ignore-errors', 'incf', 'in-package', 'lambda', 'loop',
'loop-finish', 'make-method', 'multiple-value-bind', 'multiple-value-list',
'multiple-value-setq', 'nth-value', 'or', 'pop',
'pprint-exit-if-list-exhausted', 'pprint-logical-block', 'pprint-pop',
'print-unreadable-object', 'prog', 'prog*', 'prog1', 'prog2', 'psetf',
'psetq', 'push', 'pushnew', 'remf', 'restart-bind', 'restart-case',
'return', 'rotatef', 'setf', 'shiftf', 'step', 'time', 'trace', 'typecase',
'unless', 'untrace', 'when', 'with-accessors', 'with-compilation-unit',
'with-condition-restarts', 'with-hash-table-iterator',
'with-input-from-string', 'with-open-file', 'with-open-stream',
'with-output-to-string', 'with-package-iterator', 'with-simple-restart',
'with-slots', 'with-standard-io-syntax',
))
LAMBDA_LIST_KEYWORDS = set((
'&allow-other-keys', '&aux', '&body', '&environment', '&key', '&optional',
'&rest', '&whole',
))
DECLARATIONS = set((
'dynamic-extent', 'ignore', 'optimize', 'ftype', 'inline', 'special',
'ignorable', 'notinline', 'type',
))
BUILTIN_TYPES = set((
'atom', 'boolean', 'base-char', 'base-string', 'bignum', 'bit',
'compiled-function', 'extended-char', 'fixnum', 'keyword', 'nil',
'signed-byte', 'short-float', 'single-float', 'double-float', 'long-float',
'simple-array', 'simple-base-string', 'simple-bit-vector', 'simple-string',
'simple-vector', 'standard-char', 'unsigned-byte',
# Condition Types
'arithmetic-error', 'cell-error', 'condition', 'control-error',
'division-by-zero', 'end-of-file', 'error', 'file-error',
'floating-point-inexact', 'floating-point-overflow',
'floating-point-underflow', 'floating-point-invalid-operation',
'parse-error', 'package-error', 'print-not-readable', 'program-error',
'reader-error', 'serious-condition', 'simple-condition', 'simple-error',
'simple-type-error', 'simple-warning', 'stream-error', 'storage-condition',
'style-warning', 'type-error', 'unbound-variable', 'unbound-slot',
'undefined-function', 'warning',
))
BUILTIN_CLASSES = set((
'array', 'broadcast-stream', 'bit-vector', 'built-in-class', 'character',
'class', 'complex', 'concatenated-stream', 'cons', 'echo-stream',
'file-stream', 'float', 'function', 'generic-function', 'hash-table',
'integer', 'list', 'logical-pathname', 'method-combination', 'method',
'null', 'number', 'package', 'pathname', 'ratio', 'rational', 'readtable',
'real', 'random-state', 'restart', 'sequence', 'standard-class',
'standard-generic-function', 'standard-method', 'standard-object',
'string-stream', 'stream', 'string', 'structure-class', 'structure-object',
'symbol', 'synonym-stream', 't', 'two-way-stream', 'vector',
))

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1,295 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexers._lua_builtins
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
This file contains the names and modules of lua functions
It is able to re-generate itself, but for adding new functions you
probably have to add some callbacks (see function module_callbacks).
Do not edit the MODULES dict by hand.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from __future__ import print_function
MODULES = {'basic': ('_G',
'_VERSION',
'assert',
'collectgarbage',
'dofile',
'error',
'getmetatable',
'ipairs',
'load',
'loadfile',
'next',
'pairs',
'pcall',
'print',
'rawequal',
'rawget',
'rawlen',
'rawset',
'select',
'setmetatable',
'tonumber',
'tostring',
'type',
'xpcall'),
'bit32': ('bit32.arshift',
'bit32.band',
'bit32.bnot',
'bit32.bor',
'bit32.btest',
'bit32.bxor',
'bit32.extract',
'bit32.lrotate',
'bit32.lshift',
'bit32.replace',
'bit32.rrotate',
'bit32.rshift'),
'coroutine': ('coroutine.create',
'coroutine.isyieldable',
'coroutine.resume',
'coroutine.running',
'coroutine.status',
'coroutine.wrap',
'coroutine.yield'),
'debug': ('debug.debug',
'debug.gethook',
'debug.getinfo',
'debug.getlocal',
'debug.getmetatable',
'debug.getregistry',
'debug.getupvalue',
'debug.getuservalue',
'debug.sethook',
'debug.setlocal',
'debug.setmetatable',
'debug.setupvalue',
'debug.setuservalue',
'debug.traceback',
'debug.upvalueid',
'debug.upvaluejoin'),
'io': ('io.close',
'io.flush',
'io.input',
'io.lines',
'io.open',
'io.output',
'io.popen',
'io.read',
'io.stderr',
'io.stdin',
'io.stdout',
'io.tmpfile',
'io.type',
'io.write'),
'math': ('math.abs',
'math.acos',
'math.asin',
'math.atan',
'math.atan2',
'math.ceil',
'math.cos',
'math.cosh',
'math.deg',
'math.exp',
'math.floor',
'math.fmod',
'math.frexp',
'math.huge',
'math.ldexp',
'math.log',
'math.max',
'math.maxinteger',
'math.min',
'math.mininteger',
'math.modf',
'math.pi',
'math.pow',
'math.rad',
'math.random',
'math.randomseed',
'math.sin',
'math.sinh',
'math.sqrt',
'math.tan',
'math.tanh',
'math.tointeger',
'math.type',
'math.ult'),
'modules': ('package.config',
'package.cpath',
'package.loaded',
'package.loadlib',
'package.path',
'package.preload',
'package.searchers',
'package.searchpath',
'require'),
'os': ('os.clock',
'os.date',
'os.difftime',
'os.execute',
'os.exit',
'os.getenv',
'os.remove',
'os.rename',
'os.setlocale',
'os.time',
'os.tmpname'),
'string': ('string.byte',
'string.char',
'string.dump',
'string.find',
'string.format',
'string.gmatch',
'string.gsub',
'string.len',
'string.lower',
'string.match',
'string.pack',
'string.packsize',
'string.rep',
'string.reverse',
'string.sub',
'string.unpack',
'string.upper'),
'table': ('table.concat',
'table.insert',
'table.move',
'table.pack',
'table.remove',
'table.sort',
'table.unpack'),
'utf8': ('utf8.char',
'utf8.charpattern',
'utf8.codepoint',
'utf8.codes',
'utf8.len',
'utf8.offset')}
if __name__ == '__main__': # pragma: no cover
import re
import sys
# urllib ends up wanting to import a module called 'math' -- if
# pygments/lexers is in the path, this ends badly.
for i in range(len(sys.path)-1, -1, -1):
if sys.path[i].endswith('/lexers'):
del sys.path[i]
try:
from urllib import urlopen
except ImportError:
from urllib.request import urlopen
import pprint
# you can't generally find out what module a function belongs to if you
# have only its name. Because of this, here are some callback functions
# that recognize if a gioven function belongs to a specific module
def module_callbacks():
def is_in_coroutine_module(name):
return name.startswith('coroutine.')
def is_in_modules_module(name):
if name in ['require', 'module'] or name.startswith('package'):
return True
else:
return False
def is_in_string_module(name):
return name.startswith('string.')
def is_in_table_module(name):
return name.startswith('table.')
def is_in_math_module(name):
return name.startswith('math')
def is_in_io_module(name):
return name.startswith('io.')
def is_in_os_module(name):
return name.startswith('os.')
def is_in_debug_module(name):
return name.startswith('debug.')
return {'coroutine': is_in_coroutine_module,
'modules': is_in_modules_module,
'string': is_in_string_module,
'table': is_in_table_module,
'math': is_in_math_module,
'io': is_in_io_module,
'os': is_in_os_module,
'debug': is_in_debug_module}
def get_newest_version():
f = urlopen('http://www.lua.org/manual/')
r = re.compile(r'^<A HREF="(\d\.\d)/">(Lua )?\1</A>')
for line in f:
m = r.match(line)
if m is not None:
return m.groups()[0]
def get_lua_functions(version):
f = urlopen('http://www.lua.org/manual/%s/' % version)
r = re.compile(r'^<A HREF="manual.html#pdf-(?!lua|LUA)([^:]+)">\1</A>')
functions = []
for line in f:
m = r.match(line)
if m is not None:
functions.append(m.groups()[0])
return functions
def get_function_module(name):
for mod, cb in module_callbacks().items():
if cb(name):
return mod
if '.' in name:
return name.split('.')[0]
else:
return 'basic'
def regenerate(filename, modules):
with open(filename) as fp:
content = fp.read()
header = content[:content.find('MODULES = {')]
footer = content[content.find("if __name__ == '__main__':"):]
with open(filename, 'w') as fp:
fp.write(header)
fp.write('MODULES = %s\n\n' % pprint.pformat(modules))
fp.write(footer)
def run():
version = get_newest_version()
functions = set()
for v in ('5.2', version):
print('> Downloading function index for Lua %s' % v)
f = get_lua_functions(v)
print('> %d functions found, %d new:' %
(len(f), len(set(f) - functions)))
functions |= set(f)
functions = sorted(functions)
modules = {}
for full_function_name in functions:
print('>> %s' % full_function_name)
m = get_function_module(full_function_name)
modules.setdefault(m, []).append(full_function_name)
modules = {k: tuple(v) for k, v in modules.iteritems()}
regenerate(__file__, modules)
run()

View File

@ -1,501 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexers._mapping
~~~~~~~~~~~~~~~~~~~~~~~~
Lexer mapping definitions. This file is generated by itself. Everytime
you change something on a builtin lexer definition, run this script from
the lexers folder to update it.
Do not alter the LEXERS dictionary by hand.
:copyright: Copyright 2006-2014, 2016 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from __future__ import print_function
LEXERS = {
'ABAPLexer': ('pygments.lexers.business', 'ABAP', ('abap',), ('*.abap', '*.ABAP'), ('text/x-abap',)),
'APLLexer': ('pygments.lexers.apl', 'APL', ('apl',), ('*.apl',), ()),
'AbnfLexer': ('pygments.lexers.grammar_notation', 'ABNF', ('abnf',), ('*.abnf',), ('text/x-abnf',)),
'ActionScript3Lexer': ('pygments.lexers.actionscript', 'ActionScript 3', ('as3', 'actionscript3'), ('*.as',), ('application/x-actionscript3', 'text/x-actionscript3', 'text/actionscript3')),
'ActionScriptLexer': ('pygments.lexers.actionscript', 'ActionScript', ('as', 'actionscript'), ('*.as',), ('application/x-actionscript', 'text/x-actionscript', 'text/actionscript')),
'AdaLexer': ('pygments.lexers.pascal', 'Ada', ('ada', 'ada95', 'ada2005'), ('*.adb', '*.ads', '*.ada'), ('text/x-ada',)),
'AdlLexer': ('pygments.lexers.archetype', 'ADL', ('adl',), ('*.adl', '*.adls', '*.adlf', '*.adlx'), ()),
'AgdaLexer': ('pygments.lexers.haskell', 'Agda', ('agda',), ('*.agda',), ('text/x-agda',)),
'AheuiLexer': ('pygments.lexers.esoteric', 'Aheui', ('aheui',), ('*.aheui',), ()),
'AlloyLexer': ('pygments.lexers.dsls', 'Alloy', ('alloy',), ('*.als',), ('text/x-alloy',)),
'AmbientTalkLexer': ('pygments.lexers.ambient', 'AmbientTalk', ('at', 'ambienttalk', 'ambienttalk/2'), ('*.at',), ('text/x-ambienttalk',)),
'AmplLexer': ('pygments.lexers.ampl', 'Ampl', ('ampl',), ('*.run',), ()),
'Angular2HtmlLexer': ('pygments.lexers.templates', 'HTML + Angular2', ('html+ng2',), ('*.ng2',), ()),
'Angular2Lexer': ('pygments.lexers.templates', 'Angular2', ('ng2',), (), ()),
'AntlrActionScriptLexer': ('pygments.lexers.parsers', 'ANTLR With ActionScript Target', ('antlr-as', 'antlr-actionscript'), ('*.G', '*.g'), ()),
'AntlrCSharpLexer': ('pygments.lexers.parsers', 'ANTLR With C# Target', ('antlr-csharp', 'antlr-c#'), ('*.G', '*.g'), ()),
'AntlrCppLexer': ('pygments.lexers.parsers', 'ANTLR With CPP Target', ('antlr-cpp',), ('*.G', '*.g'), ()),
'AntlrJavaLexer': ('pygments.lexers.parsers', 'ANTLR With Java Target', ('antlr-java',), ('*.G', '*.g'), ()),
'AntlrLexer': ('pygments.lexers.parsers', 'ANTLR', ('antlr',), (), ()),
'AntlrObjectiveCLexer': ('pygments.lexers.parsers', 'ANTLR With ObjectiveC Target', ('antlr-objc',), ('*.G', '*.g'), ()),
'AntlrPerlLexer': ('pygments.lexers.parsers', 'ANTLR With Perl Target', ('antlr-perl',), ('*.G', '*.g'), ()),
'AntlrPythonLexer': ('pygments.lexers.parsers', 'ANTLR With Python Target', ('antlr-python',), ('*.G', '*.g'), ()),
'AntlrRubyLexer': ('pygments.lexers.parsers', 'ANTLR With Ruby Target', ('antlr-ruby', 'antlr-rb'), ('*.G', '*.g'), ()),
'ApacheConfLexer': ('pygments.lexers.configs', 'ApacheConf', ('apacheconf', 'aconf', 'apache'), ('.htaccess', 'apache.conf', 'apache2.conf'), ('text/x-apacheconf',)),
'AppleScriptLexer': ('pygments.lexers.scripting', 'AppleScript', ('applescript',), ('*.applescript',), ()),
'ArduinoLexer': ('pygments.lexers.c_like', 'Arduino', ('arduino',), ('*.ino',), ('text/x-arduino',)),
'AspectJLexer': ('pygments.lexers.jvm', 'AspectJ', ('aspectj',), ('*.aj',), ('text/x-aspectj',)),
'AsymptoteLexer': ('pygments.lexers.graphics', 'Asymptote', ('asy', 'asymptote'), ('*.asy',), ('text/x-asymptote',)),
'AutoItLexer': ('pygments.lexers.automation', 'AutoIt', ('autoit',), ('*.au3',), ('text/x-autoit',)),
'AutohotkeyLexer': ('pygments.lexers.automation', 'autohotkey', ('ahk', 'autohotkey'), ('*.ahk', '*.ahkl'), ('text/x-autohotkey',)),
'AwkLexer': ('pygments.lexers.textedit', 'Awk', ('awk', 'gawk', 'mawk', 'nawk'), ('*.awk',), ('application/x-awk',)),
'BBCodeLexer': ('pygments.lexers.markup', 'BBCode', ('bbcode',), (), ('text/x-bbcode',)),
'BCLexer': ('pygments.lexers.algebra', 'BC', ('bc',), ('*.bc',), ()),
'BSTLexer': ('pygments.lexers.bibtex', 'BST', ('bst', 'bst-pybtex'), ('*.bst',), ()),
'BaseMakefileLexer': ('pygments.lexers.make', 'Base Makefile', ('basemake',), (), ()),
'BashLexer': ('pygments.lexers.shell', 'Bash', ('bash', 'sh', 'ksh', 'zsh', 'shell'), ('*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass', '*.exheres-0', '*.exlib', '*.zsh', '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'zshrc', '.zshrc', 'PKGBUILD'), ('application/x-sh', 'application/x-shellscript')),
'BashSessionLexer': ('pygments.lexers.shell', 'Bash Session', ('console', 'shell-session'), ('*.sh-session', '*.shell-session'), ('application/x-shell-session', 'application/x-sh-session')),
'BatchLexer': ('pygments.lexers.shell', 'Batchfile', ('bat', 'batch', 'dosbatch', 'winbatch'), ('*.bat', '*.cmd'), ('application/x-dos-batch',)),
'BefungeLexer': ('pygments.lexers.esoteric', 'Befunge', ('befunge',), ('*.befunge',), ('application/x-befunge',)),
'BibTeXLexer': ('pygments.lexers.bibtex', 'BibTeX', ('bib', 'bibtex'), ('*.bib',), ('text/x-bibtex',)),
'BlitzBasicLexer': ('pygments.lexers.basic', 'BlitzBasic', ('blitzbasic', 'b3d', 'bplus'), ('*.bb', '*.decls'), ('text/x-bb',)),
'BlitzMaxLexer': ('pygments.lexers.basic', 'BlitzMax', ('blitzmax', 'bmax'), ('*.bmx',), ('text/x-bmx',)),
'BnfLexer': ('pygments.lexers.grammar_notation', 'BNF', ('bnf',), ('*.bnf',), ('text/x-bnf',)),
'BooLexer': ('pygments.lexers.dotnet', 'Boo', ('boo',), ('*.boo',), ('text/x-boo',)),
'BoogieLexer': ('pygments.lexers.verification', 'Boogie', ('boogie',), ('*.bpl',), ()),
'BrainfuckLexer': ('pygments.lexers.esoteric', 'Brainfuck', ('brainfuck', 'bf'), ('*.bf', '*.b'), ('application/x-brainfuck',)),
'BroLexer': ('pygments.lexers.dsls', 'Bro', ('bro',), ('*.bro',), ()),
'BugsLexer': ('pygments.lexers.modeling', 'BUGS', ('bugs', 'winbugs', 'openbugs'), ('*.bug',), ()),
'CAmkESLexer': ('pygments.lexers.esoteric', 'CAmkES', ('camkes', 'idl4'), ('*.camkes', '*.idl4'), ()),
'CLexer': ('pygments.lexers.c_cpp', 'C', ('c',), ('*.c', '*.h', '*.idc'), ('text/x-chdr', 'text/x-csrc')),
'CMakeLexer': ('pygments.lexers.make', 'CMake', ('cmake',), ('*.cmake', 'CMakeLists.txt'), ('text/x-cmake',)),
'CObjdumpLexer': ('pygments.lexers.asm', 'c-objdump', ('c-objdump',), ('*.c-objdump',), ('text/x-c-objdump',)),
'CPSALexer': ('pygments.lexers.lisp', 'CPSA', ('cpsa',), ('*.cpsa',), ()),
'CSharpAspxLexer': ('pygments.lexers.dotnet', 'aspx-cs', ('aspx-cs',), ('*.aspx', '*.asax', '*.ascx', '*.ashx', '*.asmx', '*.axd'), ()),
'CSharpLexer': ('pygments.lexers.dotnet', 'C#', ('csharp', 'c#'), ('*.cs',), ('text/x-csharp',)),
'Ca65Lexer': ('pygments.lexers.asm', 'ca65 assembler', ('ca65',), ('*.s',), ()),
'CadlLexer': ('pygments.lexers.archetype', 'cADL', ('cadl',), ('*.cadl',), ()),
'CapDLLexer': ('pygments.lexers.esoteric', 'CapDL', ('capdl',), ('*.cdl',), ()),
'CapnProtoLexer': ('pygments.lexers.capnproto', "Cap'n Proto", ('capnp',), ('*.capnp',), ()),
'CbmBasicV2Lexer': ('pygments.lexers.basic', 'CBM BASIC V2', ('cbmbas',), ('*.bas',), ()),
'CeylonLexer': ('pygments.lexers.jvm', 'Ceylon', ('ceylon',), ('*.ceylon',), ('text/x-ceylon',)),
'Cfengine3Lexer': ('pygments.lexers.configs', 'CFEngine3', ('cfengine3', 'cf3'), ('*.cf',), ()),
'ChaiscriptLexer': ('pygments.lexers.scripting', 'ChaiScript', ('chai', 'chaiscript'), ('*.chai',), ('text/x-chaiscript', 'application/x-chaiscript')),
'ChapelLexer': ('pygments.lexers.chapel', 'Chapel', ('chapel', 'chpl'), ('*.chpl',), ()),
'CheetahHtmlLexer': ('pygments.lexers.templates', 'HTML+Cheetah', ('html+cheetah', 'html+spitfire', 'htmlcheetah'), (), ('text/html+cheetah', 'text/html+spitfire')),
'CheetahJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Cheetah', ('js+cheetah', 'javascript+cheetah', 'js+spitfire', 'javascript+spitfire'), (), ('application/x-javascript+cheetah', 'text/x-javascript+cheetah', 'text/javascript+cheetah', 'application/x-javascript+spitfire', 'text/x-javascript+spitfire', 'text/javascript+spitfire')),
'CheetahLexer': ('pygments.lexers.templates', 'Cheetah', ('cheetah', 'spitfire'), ('*.tmpl', '*.spt'), ('application/x-cheetah', 'application/x-spitfire')),
'CheetahXmlLexer': ('pygments.lexers.templates', 'XML+Cheetah', ('xml+cheetah', 'xml+spitfire'), (), ('application/xml+cheetah', 'application/xml+spitfire')),
'CirruLexer': ('pygments.lexers.webmisc', 'Cirru', ('cirru',), ('*.cirru',), ('text/x-cirru',)),
'ClayLexer': ('pygments.lexers.c_like', 'Clay', ('clay',), ('*.clay',), ('text/x-clay',)),
'CleanLexer': ('pygments.lexers.clean', 'Clean', ('clean',), ('*.icl', '*.dcl'), ()),
'ClojureLexer': ('pygments.lexers.jvm', 'Clojure', ('clojure', 'clj'), ('*.clj',), ('text/x-clojure', 'application/x-clojure')),
'ClojureScriptLexer': ('pygments.lexers.jvm', 'ClojureScript', ('clojurescript', 'cljs'), ('*.cljs',), ('text/x-clojurescript', 'application/x-clojurescript')),
'CobolFreeformatLexer': ('pygments.lexers.business', 'COBOLFree', ('cobolfree',), ('*.cbl', '*.CBL'), ()),
'CobolLexer': ('pygments.lexers.business', 'COBOL', ('cobol',), ('*.cob', '*.COB', '*.cpy', '*.CPY'), ('text/x-cobol',)),
'CoffeeScriptLexer': ('pygments.lexers.javascript', 'CoffeeScript', ('coffee-script', 'coffeescript', 'coffee'), ('*.coffee',), ('text/coffeescript',)),
'ColdfusionCFCLexer': ('pygments.lexers.templates', 'Coldfusion CFC', ('cfc',), ('*.cfc',), ()),
'ColdfusionHtmlLexer': ('pygments.lexers.templates', 'Coldfusion HTML', ('cfm',), ('*.cfm', '*.cfml'), ('application/x-coldfusion',)),
'ColdfusionLexer': ('pygments.lexers.templates', 'cfstatement', ('cfs',), (), ()),
'CommonLispLexer': ('pygments.lexers.lisp', 'Common Lisp', ('common-lisp', 'cl', 'lisp'), ('*.cl', '*.lisp'), ('text/x-common-lisp',)),
'ComponentPascalLexer': ('pygments.lexers.oberon', 'Component Pascal', ('componentpascal', 'cp'), ('*.cp', '*.cps'), ('text/x-component-pascal',)),
'CoqLexer': ('pygments.lexers.theorem', 'Coq', ('coq',), ('*.v',), ('text/x-coq',)),
'CppLexer': ('pygments.lexers.c_cpp', 'C++', ('cpp', 'c++'), ('*.cpp', '*.hpp', '*.c++', '*.h++', '*.cc', '*.hh', '*.cxx', '*.hxx', '*.C', '*.H', '*.cp', '*.CPP'), ('text/x-c++hdr', 'text/x-c++src')),
'CppObjdumpLexer': ('pygments.lexers.asm', 'cpp-objdump', ('cpp-objdump', 'c++-objdumb', 'cxx-objdump'), ('*.cpp-objdump', '*.c++-objdump', '*.cxx-objdump'), ('text/x-cpp-objdump',)),
'CrmshLexer': ('pygments.lexers.dsls', 'Crmsh', ('crmsh', 'pcmk'), ('*.crmsh', '*.pcmk'), ()),
'CrocLexer': ('pygments.lexers.d', 'Croc', ('croc',), ('*.croc',), ('text/x-crocsrc',)),
'CryptolLexer': ('pygments.lexers.haskell', 'Cryptol', ('cryptol', 'cry'), ('*.cry',), ('text/x-cryptol',)),
'CrystalLexer': ('pygments.lexers.crystal', 'Crystal', ('cr', 'crystal'), ('*.cr',), ('text/x-crystal',)),
'CsoundDocumentLexer': ('pygments.lexers.csound', 'Csound Document', ('csound-document', 'csound-csd'), ('*.csd',), ()),
'CsoundOrchestraLexer': ('pygments.lexers.csound', 'Csound Orchestra', ('csound', 'csound-orc'), ('*.orc',), ()),
'CsoundScoreLexer': ('pygments.lexers.csound', 'Csound Score', ('csound-score', 'csound-sco'), ('*.sco',), ()),
'CssDjangoLexer': ('pygments.lexers.templates', 'CSS+Django/Jinja', ('css+django', 'css+jinja'), (), ('text/css+django', 'text/css+jinja')),
'CssErbLexer': ('pygments.lexers.templates', 'CSS+Ruby', ('css+erb', 'css+ruby'), (), ('text/css+ruby',)),
'CssGenshiLexer': ('pygments.lexers.templates', 'CSS+Genshi Text', ('css+genshitext', 'css+genshi'), (), ('text/css+genshi',)),
'CssLexer': ('pygments.lexers.css', 'CSS', ('css',), ('*.css',), ('text/css',)),
'CssPhpLexer': ('pygments.lexers.templates', 'CSS+PHP', ('css+php',), (), ('text/css+php',)),
'CssSmartyLexer': ('pygments.lexers.templates', 'CSS+Smarty', ('css+smarty',), (), ('text/css+smarty',)),
'CudaLexer': ('pygments.lexers.c_like', 'CUDA', ('cuda', 'cu'), ('*.cu', '*.cuh'), ('text/x-cuda',)),
'CypherLexer': ('pygments.lexers.graph', 'Cypher', ('cypher',), ('*.cyp', '*.cypher'), ()),
'CythonLexer': ('pygments.lexers.python', 'Cython', ('cython', 'pyx', 'pyrex'), ('*.pyx', '*.pxd', '*.pxi'), ('text/x-cython', 'application/x-cython')),
'DLexer': ('pygments.lexers.d', 'D', ('d',), ('*.d', '*.di'), ('text/x-dsrc',)),
'DObjdumpLexer': ('pygments.lexers.asm', 'd-objdump', ('d-objdump',), ('*.d-objdump',), ('text/x-d-objdump',)),
'DarcsPatchLexer': ('pygments.lexers.diff', 'Darcs Patch', ('dpatch',), ('*.dpatch', '*.darcspatch'), ()),
'DartLexer': ('pygments.lexers.javascript', 'Dart', ('dart',), ('*.dart',), ('text/x-dart',)),
'DebianControlLexer': ('pygments.lexers.installers', 'Debian Control file', ('control', 'debcontrol'), ('control',), ()),
'DelphiLexer': ('pygments.lexers.pascal', 'Delphi', ('delphi', 'pas', 'pascal', 'objectpascal'), ('*.pas', '*.dpr'), ('text/x-pascal',)),
'DgLexer': ('pygments.lexers.python', 'dg', ('dg',), ('*.dg',), ('text/x-dg',)),
'DiffLexer': ('pygments.lexers.diff', 'Diff', ('diff', 'udiff'), ('*.diff', '*.patch'), ('text/x-diff', 'text/x-patch')),
'DjangoLexer': ('pygments.lexers.templates', 'Django/Jinja', ('django', 'jinja'), (), ('application/x-django-templating', 'application/x-jinja')),
'DockerLexer': ('pygments.lexers.configs', 'Docker', ('docker', 'dockerfile'), ('Dockerfile', '*.docker'), ('text/x-dockerfile-config',)),
'DtdLexer': ('pygments.lexers.html', 'DTD', ('dtd',), ('*.dtd',), ('application/xml-dtd',)),
'DuelLexer': ('pygments.lexers.webmisc', 'Duel', ('duel', 'jbst', 'jsonml+bst'), ('*.duel', '*.jbst'), ('text/x-duel', 'text/x-jbst')),
'DylanConsoleLexer': ('pygments.lexers.dylan', 'Dylan session', ('dylan-console', 'dylan-repl'), ('*.dylan-console',), ('text/x-dylan-console',)),
'DylanLexer': ('pygments.lexers.dylan', 'Dylan', ('dylan',), ('*.dylan', '*.dyl', '*.intr'), ('text/x-dylan',)),
'DylanLidLexer': ('pygments.lexers.dylan', 'DylanLID', ('dylan-lid', 'lid'), ('*.lid', '*.hdp'), ('text/x-dylan-lid',)),
'ECLLexer': ('pygments.lexers.ecl', 'ECL', ('ecl',), ('*.ecl',), ('application/x-ecl',)),
'ECLexer': ('pygments.lexers.c_like', 'eC', ('ec',), ('*.ec', '*.eh'), ('text/x-echdr', 'text/x-ecsrc')),
'EarlGreyLexer': ('pygments.lexers.javascript', 'Earl Grey', ('earl-grey', 'earlgrey', 'eg'), ('*.eg',), ('text/x-earl-grey',)),
'EasytrieveLexer': ('pygments.lexers.scripting', 'Easytrieve', ('easytrieve',), ('*.ezt', '*.mac'), ('text/x-easytrieve',)),
'EbnfLexer': ('pygments.lexers.parsers', 'EBNF', ('ebnf',), ('*.ebnf',), ('text/x-ebnf',)),
'EiffelLexer': ('pygments.lexers.eiffel', 'Eiffel', ('eiffel',), ('*.e',), ('text/x-eiffel',)),
'ElixirConsoleLexer': ('pygments.lexers.erlang', 'Elixir iex session', ('iex',), (), ('text/x-elixir-shellsession',)),
'ElixirLexer': ('pygments.lexers.erlang', 'Elixir', ('elixir', 'ex', 'exs'), ('*.ex', '*.exs'), ('text/x-elixir',)),
'ElmLexer': ('pygments.lexers.elm', 'Elm', ('elm',), ('*.elm',), ('text/x-elm',)),
'EmacsLispLexer': ('pygments.lexers.lisp', 'EmacsLisp', ('emacs', 'elisp', 'emacs-lisp'), ('*.el',), ('text/x-elisp', 'application/x-elisp')),
'ErbLexer': ('pygments.lexers.templates', 'ERB', ('erb',), (), ('application/x-ruby-templating',)),
'ErlangLexer': ('pygments.lexers.erlang', 'Erlang', ('erlang',), ('*.erl', '*.hrl', '*.es', '*.escript'), ('text/x-erlang',)),
'ErlangShellLexer': ('pygments.lexers.erlang', 'Erlang erl session', ('erl',), ('*.erl-sh',), ('text/x-erl-shellsession',)),
'EvoqueHtmlLexer': ('pygments.lexers.templates', 'HTML+Evoque', ('html+evoque',), ('*.html',), ('text/html+evoque',)),
'EvoqueLexer': ('pygments.lexers.templates', 'Evoque', ('evoque',), ('*.evoque',), ('application/x-evoque',)),
'EvoqueXmlLexer': ('pygments.lexers.templates', 'XML+Evoque', ('xml+evoque',), ('*.xml',), ('application/xml+evoque',)),
'EzhilLexer': ('pygments.lexers.ezhil', 'Ezhil', ('ezhil',), ('*.n',), ('text/x-ezhil',)),
'FSharpLexer': ('pygments.lexers.dotnet', 'F#', ('fsharp',), ('*.fs', '*.fsi'), ('text/x-fsharp',)),
'FactorLexer': ('pygments.lexers.factor', 'Factor', ('factor',), ('*.factor',), ('text/x-factor',)),
'FancyLexer': ('pygments.lexers.ruby', 'Fancy', ('fancy', 'fy'), ('*.fy', '*.fancypack'), ('text/x-fancysrc',)),
'FantomLexer': ('pygments.lexers.fantom', 'Fantom', ('fan',), ('*.fan',), ('application/x-fantom',)),
'FelixLexer': ('pygments.lexers.felix', 'Felix', ('felix', 'flx'), ('*.flx', '*.flxh'), ('text/x-felix',)),
'FishShellLexer': ('pygments.lexers.shell', 'Fish', ('fish', 'fishshell'), ('*.fish', '*.load'), ('application/x-fish',)),
'FlatlineLexer': ('pygments.lexers.dsls', 'Flatline', ('flatline',), (), ('text/x-flatline',)),
'ForthLexer': ('pygments.lexers.forth', 'Forth', ('forth',), ('*.frt', '*.fs'), ('application/x-forth',)),
'FortranFixedLexer': ('pygments.lexers.fortran', 'FortranFixed', ('fortranfixed',), ('*.f', '*.F'), ()),
'FortranLexer': ('pygments.lexers.fortran', 'Fortran', ('fortran',), ('*.f03', '*.f90', '*.F03', '*.F90'), ('text/x-fortran',)),
'FoxProLexer': ('pygments.lexers.foxpro', 'FoxPro', ('foxpro', 'vfp', 'clipper', 'xbase'), ('*.PRG', '*.prg'), ()),
'GAPLexer': ('pygments.lexers.algebra', 'GAP', ('gap',), ('*.g', '*.gd', '*.gi', '*.gap'), ()),
'GLShaderLexer': ('pygments.lexers.graphics', 'GLSL', ('glsl',), ('*.vert', '*.frag', '*.geo'), ('text/x-glslsrc',)),
'GasLexer': ('pygments.lexers.asm', 'GAS', ('gas', 'asm'), ('*.s', '*.S'), ('text/x-gas',)),
'GenshiLexer': ('pygments.lexers.templates', 'Genshi', ('genshi', 'kid', 'xml+genshi', 'xml+kid'), ('*.kid',), ('application/x-genshi', 'application/x-kid')),
'GenshiTextLexer': ('pygments.lexers.templates', 'Genshi Text', ('genshitext',), (), ('application/x-genshi-text', 'text/x-genshi')),
'GettextLexer': ('pygments.lexers.textfmts', 'Gettext Catalog', ('pot', 'po'), ('*.pot', '*.po'), ('application/x-gettext', 'text/x-gettext', 'text/gettext')),
'GherkinLexer': ('pygments.lexers.testing', 'Gherkin', ('cucumber', 'gherkin'), ('*.feature',), ('text/x-gherkin',)),
'GnuplotLexer': ('pygments.lexers.graphics', 'Gnuplot', ('gnuplot',), ('*.plot', '*.plt'), ('text/x-gnuplot',)),
'GoLexer': ('pygments.lexers.go', 'Go', ('go',), ('*.go',), ('text/x-gosrc',)),
'GoloLexer': ('pygments.lexers.jvm', 'Golo', ('golo',), ('*.golo',), ()),
'GoodDataCLLexer': ('pygments.lexers.business', 'GoodData-CL', ('gooddata-cl',), ('*.gdc',), ('text/x-gooddata-cl',)),
'GosuLexer': ('pygments.lexers.jvm', 'Gosu', ('gosu',), ('*.gs', '*.gsx', '*.gsp', '*.vark'), ('text/x-gosu',)),
'GosuTemplateLexer': ('pygments.lexers.jvm', 'Gosu Template', ('gst',), ('*.gst',), ('text/x-gosu-template',)),
'GroffLexer': ('pygments.lexers.markup', 'Groff', ('groff', 'nroff', 'man'), ('*.[1234567]', '*.man'), ('application/x-troff', 'text/troff')),
'GroovyLexer': ('pygments.lexers.jvm', 'Groovy', ('groovy',), ('*.groovy', '*.gradle'), ('text/x-groovy',)),
'HamlLexer': ('pygments.lexers.html', 'Haml', ('haml',), ('*.haml',), ('text/x-haml',)),
'HandlebarsHtmlLexer': ('pygments.lexers.templates', 'HTML+Handlebars', ('html+handlebars',), ('*.handlebars', '*.hbs'), ('text/html+handlebars', 'text/x-handlebars-template')),
'HandlebarsLexer': ('pygments.lexers.templates', 'Handlebars', ('handlebars',), (), ()),
'HaskellLexer': ('pygments.lexers.haskell', 'Haskell', ('haskell', 'hs'), ('*.hs',), ('text/x-haskell',)),
'HaxeLexer': ('pygments.lexers.haxe', 'Haxe', ('hx', 'haxe', 'hxsl'), ('*.hx', '*.hxsl'), ('text/haxe', 'text/x-haxe', 'text/x-hx')),
'HexdumpLexer': ('pygments.lexers.hexdump', 'Hexdump', ('hexdump',), (), ()),
'HsailLexer': ('pygments.lexers.asm', 'HSAIL', ('hsail', 'hsa'), ('*.hsail',), ('text/x-hsail',)),
'HtmlDjangoLexer': ('pygments.lexers.templates', 'HTML+Django/Jinja', ('html+django', 'html+jinja', 'htmldjango'), (), ('text/html+django', 'text/html+jinja')),
'HtmlGenshiLexer': ('pygments.lexers.templates', 'HTML+Genshi', ('html+genshi', 'html+kid'), (), ('text/html+genshi',)),
'HtmlLexer': ('pygments.lexers.html', 'HTML', ('html',), ('*.html', '*.htm', '*.xhtml', '*.xslt'), ('text/html', 'application/xhtml+xml')),
'HtmlPhpLexer': ('pygments.lexers.templates', 'HTML+PHP', ('html+php',), ('*.phtml',), ('application/x-php', 'application/x-httpd-php', 'application/x-httpd-php3', 'application/x-httpd-php4', 'application/x-httpd-php5')),
'HtmlSmartyLexer': ('pygments.lexers.templates', 'HTML+Smarty', ('html+smarty',), (), ('text/html+smarty',)),
'HttpLexer': ('pygments.lexers.textfmts', 'HTTP', ('http',), (), ()),
'HxmlLexer': ('pygments.lexers.haxe', 'Hxml', ('haxeml', 'hxml'), ('*.hxml',), ()),
'HyLexer': ('pygments.lexers.lisp', 'Hy', ('hylang',), ('*.hy',), ('text/x-hy', 'application/x-hy')),
'HybrisLexer': ('pygments.lexers.scripting', 'Hybris', ('hybris', 'hy'), ('*.hy', '*.hyb'), ('text/x-hybris', 'application/x-hybris')),
'IDLLexer': ('pygments.lexers.idl', 'IDL', ('idl',), ('*.pro',), ('text/idl',)),
'IdrisLexer': ('pygments.lexers.haskell', 'Idris', ('idris', 'idr'), ('*.idr',), ('text/x-idris',)),
'IgorLexer': ('pygments.lexers.igor', 'Igor', ('igor', 'igorpro'), ('*.ipf',), ('text/ipf',)),
'Inform6Lexer': ('pygments.lexers.int_fiction', 'Inform 6', ('inform6', 'i6'), ('*.inf',), ()),
'Inform6TemplateLexer': ('pygments.lexers.int_fiction', 'Inform 6 template', ('i6t',), ('*.i6t',), ()),
'Inform7Lexer': ('pygments.lexers.int_fiction', 'Inform 7', ('inform7', 'i7'), ('*.ni', '*.i7x'), ()),
'IniLexer': ('pygments.lexers.configs', 'INI', ('ini', 'cfg', 'dosini'), ('*.ini', '*.cfg', '*.inf'), ('text/x-ini', 'text/inf')),
'IoLexer': ('pygments.lexers.iolang', 'Io', ('io',), ('*.io',), ('text/x-iosrc',)),
'IokeLexer': ('pygments.lexers.jvm', 'Ioke', ('ioke', 'ik'), ('*.ik',), ('text/x-iokesrc',)),
'IrcLogsLexer': ('pygments.lexers.textfmts', 'IRC logs', ('irc',), ('*.weechatlog',), ('text/x-irclog',)),
'IsabelleLexer': ('pygments.lexers.theorem', 'Isabelle', ('isabelle',), ('*.thy',), ('text/x-isabelle',)),
'JLexer': ('pygments.lexers.j', 'J', ('j',), ('*.ijs',), ('text/x-j',)),
'JagsLexer': ('pygments.lexers.modeling', 'JAGS', ('jags',), ('*.jag', '*.bug'), ()),
'JasminLexer': ('pygments.lexers.jvm', 'Jasmin', ('jasmin', 'jasminxt'), ('*.j',), ()),
'JavaLexer': ('pygments.lexers.jvm', 'Java', ('java',), ('*.java',), ('text/x-java',)),
'JavascriptDjangoLexer': ('pygments.lexers.templates', 'JavaScript+Django/Jinja', ('js+django', 'javascript+django', 'js+jinja', 'javascript+jinja'), (), ('application/x-javascript+django', 'application/x-javascript+jinja', 'text/x-javascript+django', 'text/x-javascript+jinja', 'text/javascript+django', 'text/javascript+jinja')),
'JavascriptErbLexer': ('pygments.lexers.templates', 'JavaScript+Ruby', ('js+erb', 'javascript+erb', 'js+ruby', 'javascript+ruby'), (), ('application/x-javascript+ruby', 'text/x-javascript+ruby', 'text/javascript+ruby')),
'JavascriptGenshiLexer': ('pygments.lexers.templates', 'JavaScript+Genshi Text', ('js+genshitext', 'js+genshi', 'javascript+genshitext', 'javascript+genshi'), (), ('application/x-javascript+genshi', 'text/x-javascript+genshi', 'text/javascript+genshi')),
'JavascriptLexer': ('pygments.lexers.javascript', 'JavaScript', ('js', 'javascript'), ('*.js', '*.jsm'), ('application/javascript', 'application/x-javascript', 'text/x-javascript', 'text/javascript')),
'JavascriptPhpLexer': ('pygments.lexers.templates', 'JavaScript+PHP', ('js+php', 'javascript+php'), (), ('application/x-javascript+php', 'text/x-javascript+php', 'text/javascript+php')),
'JavascriptSmartyLexer': ('pygments.lexers.templates', 'JavaScript+Smarty', ('js+smarty', 'javascript+smarty'), (), ('application/x-javascript+smarty', 'text/x-javascript+smarty', 'text/javascript+smarty')),
'JclLexer': ('pygments.lexers.scripting', 'JCL', ('jcl',), ('*.jcl',), ('text/x-jcl',)),
'JsgfLexer': ('pygments.lexers.grammar_notation', 'JSGF', ('jsgf',), ('*.jsgf',), ('application/jsgf', 'application/x-jsgf', 'text/jsgf')),
'JsonBareObjectLexer': ('pygments.lexers.data', 'JSONBareObject', ('json-object',), (), ('application/json-object',)),
'JsonLdLexer': ('pygments.lexers.data', 'JSON-LD', ('jsonld', 'json-ld'), ('*.jsonld',), ('application/ld+json',)),
'JsonLexer': ('pygments.lexers.data', 'JSON', ('json',), ('*.json',), ('application/json',)),
'JspLexer': ('pygments.lexers.templates', 'Java Server Page', ('jsp',), ('*.jsp',), ('application/x-jsp',)),
'JuliaConsoleLexer': ('pygments.lexers.julia', 'Julia console', ('jlcon',), (), ()),
'JuliaLexer': ('pygments.lexers.julia', 'Julia', ('julia', 'jl'), ('*.jl',), ('text/x-julia', 'application/x-julia')),
'JuttleLexer': ('pygments.lexers.javascript', 'Juttle', ('juttle', 'juttle'), ('*.juttle',), ('application/juttle', 'application/x-juttle', 'text/x-juttle', 'text/juttle')),
'KalLexer': ('pygments.lexers.javascript', 'Kal', ('kal',), ('*.kal',), ('text/kal', 'application/kal')),
'KconfigLexer': ('pygments.lexers.configs', 'Kconfig', ('kconfig', 'menuconfig', 'linux-config', 'kernel-config'), ('Kconfig', '*Config.in*', 'external.in*', 'standard-modules.in'), ('text/x-kconfig',)),
'KokaLexer': ('pygments.lexers.haskell', 'Koka', ('koka',), ('*.kk', '*.kki'), ('text/x-koka',)),
'KotlinLexer': ('pygments.lexers.jvm', 'Kotlin', ('kotlin',), ('*.kt',), ('text/x-kotlin',)),
'LSLLexer': ('pygments.lexers.scripting', 'LSL', ('lsl',), ('*.lsl',), ('text/x-lsl',)),
'LassoCssLexer': ('pygments.lexers.templates', 'CSS+Lasso', ('css+lasso',), (), ('text/css+lasso',)),
'LassoHtmlLexer': ('pygments.lexers.templates', 'HTML+Lasso', ('html+lasso',), (), ('text/html+lasso', 'application/x-httpd-lasso', 'application/x-httpd-lasso[89]')),
'LassoJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Lasso', ('js+lasso', 'javascript+lasso'), (), ('application/x-javascript+lasso', 'text/x-javascript+lasso', 'text/javascript+lasso')),
'LassoLexer': ('pygments.lexers.javascript', 'Lasso', ('lasso', 'lassoscript'), ('*.lasso', '*.lasso[89]'), ('text/x-lasso',)),
'LassoXmlLexer': ('pygments.lexers.templates', 'XML+Lasso', ('xml+lasso',), (), ('application/xml+lasso',)),
'LeanLexer': ('pygments.lexers.theorem', 'Lean', ('lean',), ('*.lean',), ('text/x-lean',)),
'LessCssLexer': ('pygments.lexers.css', 'LessCss', ('less',), ('*.less',), ('text/x-less-css',)),
'LighttpdConfLexer': ('pygments.lexers.configs', 'Lighttpd configuration file', ('lighty', 'lighttpd'), (), ('text/x-lighttpd-conf',)),
'LimboLexer': ('pygments.lexers.inferno', 'Limbo', ('limbo',), ('*.b',), ('text/limbo',)),
'LiquidLexer': ('pygments.lexers.templates', 'liquid', ('liquid',), ('*.liquid',), ()),
'LiterateAgdaLexer': ('pygments.lexers.haskell', 'Literate Agda', ('lagda', 'literate-agda'), ('*.lagda',), ('text/x-literate-agda',)),
'LiterateCryptolLexer': ('pygments.lexers.haskell', 'Literate Cryptol', ('lcry', 'literate-cryptol', 'lcryptol'), ('*.lcry',), ('text/x-literate-cryptol',)),
'LiterateHaskellLexer': ('pygments.lexers.haskell', 'Literate Haskell', ('lhs', 'literate-haskell', 'lhaskell'), ('*.lhs',), ('text/x-literate-haskell',)),
'LiterateIdrisLexer': ('pygments.lexers.haskell', 'Literate Idris', ('lidr', 'literate-idris', 'lidris'), ('*.lidr',), ('text/x-literate-idris',)),
'LiveScriptLexer': ('pygments.lexers.javascript', 'LiveScript', ('live-script', 'livescript'), ('*.ls',), ('text/livescript',)),
'LlvmLexer': ('pygments.lexers.asm', 'LLVM', ('llvm',), ('*.ll',), ('text/x-llvm',)),
'LogosLexer': ('pygments.lexers.objective', 'Logos', ('logos',), ('*.x', '*.xi', '*.xm', '*.xmi'), ('text/x-logos',)),
'LogtalkLexer': ('pygments.lexers.prolog', 'Logtalk', ('logtalk',), ('*.lgt', '*.logtalk'), ('text/x-logtalk',)),
'LuaLexer': ('pygments.lexers.scripting', 'Lua', ('lua',), ('*.lua', '*.wlua'), ('text/x-lua', 'application/x-lua')),
'MOOCodeLexer': ('pygments.lexers.scripting', 'MOOCode', ('moocode', 'moo'), ('*.moo',), ('text/x-moocode',)),
'MSDOSSessionLexer': ('pygments.lexers.shell', 'MSDOS Session', ('doscon',), (), ()),
'MakefileLexer': ('pygments.lexers.make', 'Makefile', ('make', 'makefile', 'mf', 'bsdmake'), ('*.mak', '*.mk', 'Makefile', 'makefile', 'Makefile.*', 'GNUmakefile'), ('text/x-makefile',)),
'MakoCssLexer': ('pygments.lexers.templates', 'CSS+Mako', ('css+mako',), (), ('text/css+mako',)),
'MakoHtmlLexer': ('pygments.lexers.templates', 'HTML+Mako', ('html+mako',), (), ('text/html+mako',)),
'MakoJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Mako', ('js+mako', 'javascript+mako'), (), ('application/x-javascript+mako', 'text/x-javascript+mako', 'text/javascript+mako')),
'MakoLexer': ('pygments.lexers.templates', 'Mako', ('mako',), ('*.mao',), ('application/x-mako',)),
'MakoXmlLexer': ('pygments.lexers.templates', 'XML+Mako', ('xml+mako',), (), ('application/xml+mako',)),
'MaqlLexer': ('pygments.lexers.business', 'MAQL', ('maql',), ('*.maql',), ('text/x-gooddata-maql', 'application/x-gooddata-maql')),
'MarkdownLexer': ('pygments.lexers.markup', 'markdown', ('md',), ('*.md',), ('text/x-markdown',)),
'MaskLexer': ('pygments.lexers.javascript', 'Mask', ('mask',), ('*.mask',), ('text/x-mask',)),
'MasonLexer': ('pygments.lexers.templates', 'Mason', ('mason',), ('*.m', '*.mhtml', '*.mc', '*.mi', 'autohandler', 'dhandler'), ('application/x-mason',)),
'MathematicaLexer': ('pygments.lexers.algebra', 'Mathematica', ('mathematica', 'mma', 'nb'), ('*.nb', '*.cdf', '*.nbp', '*.ma'), ('application/mathematica', 'application/vnd.wolfram.mathematica', 'application/vnd.wolfram.mathematica.package', 'application/vnd.wolfram.cdf')),
'MatlabLexer': ('pygments.lexers.matlab', 'Matlab', ('matlab',), ('*.m',), ('text/matlab',)),
'MatlabSessionLexer': ('pygments.lexers.matlab', 'Matlab session', ('matlabsession',), (), ()),
'MiniDLexer': ('pygments.lexers.d', 'MiniD', ('minid',), (), ('text/x-minidsrc',)),
'ModelicaLexer': ('pygments.lexers.modeling', 'Modelica', ('modelica',), ('*.mo',), ('text/x-modelica',)),
'Modula2Lexer': ('pygments.lexers.modula2', 'Modula-2', ('modula2', 'm2'), ('*.def', '*.mod'), ('text/x-modula2',)),
'MoinWikiLexer': ('pygments.lexers.markup', 'MoinMoin/Trac Wiki markup', ('trac-wiki', 'moin'), (), ('text/x-trac-wiki',)),
'MonkeyLexer': ('pygments.lexers.basic', 'Monkey', ('monkey',), ('*.monkey',), ('text/x-monkey',)),
'MonteLexer': ('pygments.lexers.monte', 'Monte', ('monte',), ('*.mt',), ()),
'MoonScriptLexer': ('pygments.lexers.scripting', 'MoonScript', ('moon', 'moonscript'), ('*.moon',), ('text/x-moonscript', 'application/x-moonscript')),
'MozPreprocCssLexer': ('pygments.lexers.markup', 'CSS+mozpreproc', ('css+mozpreproc',), ('*.css.in',), ()),
'MozPreprocHashLexer': ('pygments.lexers.markup', 'mozhashpreproc', ('mozhashpreproc',), (), ()),
'MozPreprocJavascriptLexer': ('pygments.lexers.markup', 'Javascript+mozpreproc', ('javascript+mozpreproc',), ('*.js.in',), ()),
'MozPreprocPercentLexer': ('pygments.lexers.markup', 'mozpercentpreproc', ('mozpercentpreproc',), (), ()),
'MozPreprocXulLexer': ('pygments.lexers.markup', 'XUL+mozpreproc', ('xul+mozpreproc',), ('*.xul.in',), ()),
'MqlLexer': ('pygments.lexers.c_like', 'MQL', ('mql', 'mq4', 'mq5', 'mql4', 'mql5'), ('*.mq4', '*.mq5', '*.mqh'), ('text/x-mql',)),
'MscgenLexer': ('pygments.lexers.dsls', 'Mscgen', ('mscgen', 'msc'), ('*.msc',), ()),
'MuPADLexer': ('pygments.lexers.algebra', 'MuPAD', ('mupad',), ('*.mu',), ()),
'MxmlLexer': ('pygments.lexers.actionscript', 'MXML', ('mxml',), ('*.mxml',), ()),
'MySqlLexer': ('pygments.lexers.sql', 'MySQL', ('mysql',), (), ('text/x-mysql',)),
'MyghtyCssLexer': ('pygments.lexers.templates', 'CSS+Myghty', ('css+myghty',), (), ('text/css+myghty',)),
'MyghtyHtmlLexer': ('pygments.lexers.templates', 'HTML+Myghty', ('html+myghty',), (), ('text/html+myghty',)),
'MyghtyJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Myghty', ('js+myghty', 'javascript+myghty'), (), ('application/x-javascript+myghty', 'text/x-javascript+myghty', 'text/javascript+mygthy')),
'MyghtyLexer': ('pygments.lexers.templates', 'Myghty', ('myghty',), ('*.myt', 'autodelegate'), ('application/x-myghty',)),
'MyghtyXmlLexer': ('pygments.lexers.templates', 'XML+Myghty', ('xml+myghty',), (), ('application/xml+myghty',)),
'NCLLexer': ('pygments.lexers.ncl', 'NCL', ('ncl',), ('*.ncl',), ('text/ncl',)),
'NSISLexer': ('pygments.lexers.installers', 'NSIS', ('nsis', 'nsi', 'nsh'), ('*.nsi', '*.nsh'), ('text/x-nsis',)),
'NasmLexer': ('pygments.lexers.asm', 'NASM', ('nasm',), ('*.asm', '*.ASM'), ('text/x-nasm',)),
'NasmObjdumpLexer': ('pygments.lexers.asm', 'objdump-nasm', ('objdump-nasm',), ('*.objdump-intel',), ('text/x-nasm-objdump',)),
'NemerleLexer': ('pygments.lexers.dotnet', 'Nemerle', ('nemerle',), ('*.n',), ('text/x-nemerle',)),
'NesCLexer': ('pygments.lexers.c_like', 'nesC', ('nesc',), ('*.nc',), ('text/x-nescsrc',)),
'NewLispLexer': ('pygments.lexers.lisp', 'NewLisp', ('newlisp',), ('*.lsp', '*.nl', '*.kif'), ('text/x-newlisp', 'application/x-newlisp')),
'NewspeakLexer': ('pygments.lexers.smalltalk', 'Newspeak', ('newspeak',), ('*.ns2',), ('text/x-newspeak',)),
'NginxConfLexer': ('pygments.lexers.configs', 'Nginx configuration file', ('nginx',), ('nginx.conf',), ('text/x-nginx-conf',)),
'NimrodLexer': ('pygments.lexers.nimrod', 'Nimrod', ('nim', 'nimrod'), ('*.nim', '*.nimrod'), ('text/x-nim',)),
'NitLexer': ('pygments.lexers.nit', 'Nit', ('nit',), ('*.nit',), ()),
'NixLexer': ('pygments.lexers.nix', 'Nix', ('nixos', 'nix'), ('*.nix',), ('text/x-nix',)),
'NuSMVLexer': ('pygments.lexers.smv', 'NuSMV', ('nusmv',), ('*.smv',), ()),
'NumPyLexer': ('pygments.lexers.python', 'NumPy', ('numpy',), (), ()),
'ObjdumpLexer': ('pygments.lexers.asm', 'objdump', ('objdump',), ('*.objdump',), ('text/x-objdump',)),
'ObjectiveCLexer': ('pygments.lexers.objective', 'Objective-C', ('objective-c', 'objectivec', 'obj-c', 'objc'), ('*.m', '*.h'), ('text/x-objective-c',)),
'ObjectiveCppLexer': ('pygments.lexers.objective', 'Objective-C++', ('objective-c++', 'objectivec++', 'obj-c++', 'objc++'), ('*.mm', '*.hh'), ('text/x-objective-c++',)),
'ObjectiveJLexer': ('pygments.lexers.javascript', 'Objective-J', ('objective-j', 'objectivej', 'obj-j', 'objj'), ('*.j',), ('text/x-objective-j',)),
'OcamlLexer': ('pygments.lexers.ml', 'OCaml', ('ocaml',), ('*.ml', '*.mli', '*.mll', '*.mly'), ('text/x-ocaml',)),
'OctaveLexer': ('pygments.lexers.matlab', 'Octave', ('octave',), ('*.m',), ('text/octave',)),
'OdinLexer': ('pygments.lexers.archetype', 'ODIN', ('odin',), ('*.odin',), ('text/odin',)),
'OocLexer': ('pygments.lexers.ooc', 'Ooc', ('ooc',), ('*.ooc',), ('text/x-ooc',)),
'OpaLexer': ('pygments.lexers.ml', 'Opa', ('opa',), ('*.opa',), ('text/x-opa',)),
'OpenEdgeLexer': ('pygments.lexers.business', 'OpenEdge ABL', ('openedge', 'abl', 'progress'), ('*.p', '*.cls'), ('text/x-openedge', 'application/x-openedge')),
'PacmanConfLexer': ('pygments.lexers.configs', 'PacmanConf', ('pacmanconf',), ('pacman.conf',), ()),
'PanLexer': ('pygments.lexers.dsls', 'Pan', ('pan',), ('*.pan',), ()),
'ParaSailLexer': ('pygments.lexers.parasail', 'ParaSail', ('parasail',), ('*.psi', '*.psl'), ('text/x-parasail',)),
'PawnLexer': ('pygments.lexers.pawn', 'Pawn', ('pawn',), ('*.p', '*.pwn', '*.inc'), ('text/x-pawn',)),
'Perl6Lexer': ('pygments.lexers.perl', 'Perl6', ('perl6', 'pl6'), ('*.pl', '*.pm', '*.nqp', '*.p6', '*.6pl', '*.p6l', '*.pl6', '*.6pm', '*.p6m', '*.pm6', '*.t'), ('text/x-perl6', 'application/x-perl6')),
'PerlLexer': ('pygments.lexers.perl', 'Perl', ('perl', 'pl'), ('*.pl', '*.pm', '*.t'), ('text/x-perl', 'application/x-perl')),
'PhpLexer': ('pygments.lexers.php', 'PHP', ('php', 'php3', 'php4', 'php5'), ('*.php', '*.php[345]', '*.inc'), ('text/x-php',)),
'PigLexer': ('pygments.lexers.jvm', 'Pig', ('pig',), ('*.pig',), ('text/x-pig',)),
'PikeLexer': ('pygments.lexers.c_like', 'Pike', ('pike',), ('*.pike', '*.pmod'), ('text/x-pike',)),
'PkgConfigLexer': ('pygments.lexers.configs', 'PkgConfig', ('pkgconfig',), ('*.pc',), ()),
'PlPgsqlLexer': ('pygments.lexers.sql', 'PL/pgSQL', ('plpgsql',), (), ('text/x-plpgsql',)),
'PostScriptLexer': ('pygments.lexers.graphics', 'PostScript', ('postscript', 'postscr'), ('*.ps', '*.eps'), ('application/postscript',)),
'PostgresConsoleLexer': ('pygments.lexers.sql', 'PostgreSQL console (psql)', ('psql', 'postgresql-console', 'postgres-console'), (), ('text/x-postgresql-psql',)),
'PostgresLexer': ('pygments.lexers.sql', 'PostgreSQL SQL dialect', ('postgresql', 'postgres'), (), ('text/x-postgresql',)),
'PovrayLexer': ('pygments.lexers.graphics', 'POVRay', ('pov',), ('*.pov', '*.inc'), ('text/x-povray',)),
'PowerShellLexer': ('pygments.lexers.shell', 'PowerShell', ('powershell', 'posh', 'ps1', 'psm1'), ('*.ps1', '*.psm1'), ('text/x-powershell',)),
'PowerShellSessionLexer': ('pygments.lexers.shell', 'PowerShell Session', ('ps1con',), (), ()),
'PraatLexer': ('pygments.lexers.praat', 'Praat', ('praat',), ('*.praat', '*.proc', '*.psc'), ()),
'PrologLexer': ('pygments.lexers.prolog', 'Prolog', ('prolog',), ('*.ecl', '*.prolog', '*.pro', '*.pl'), ('text/x-prolog',)),
'PropertiesLexer': ('pygments.lexers.configs', 'Properties', ('properties', 'jproperties'), ('*.properties',), ('text/x-java-properties',)),
'ProtoBufLexer': ('pygments.lexers.dsls', 'Protocol Buffer', ('protobuf', 'proto'), ('*.proto',), ()),
'PugLexer': ('pygments.lexers.html', 'Pug', ('pug', 'jade'), ('*.pug', '*.jade'), ('text/x-pug', 'text/x-jade')),
'PuppetLexer': ('pygments.lexers.dsls', 'Puppet', ('puppet',), ('*.pp',), ()),
'PyPyLogLexer': ('pygments.lexers.console', 'PyPy Log', ('pypylog', 'pypy'), ('*.pypylog',), ('application/x-pypylog',)),
'Python3Lexer': ('pygments.lexers.python', 'Python 3', ('python3', 'py3'), (), ('text/x-python3', 'application/x-python3')),
'Python3TracebackLexer': ('pygments.lexers.python', 'Python 3.0 Traceback', ('py3tb',), ('*.py3tb',), ('text/x-python3-traceback',)),
'PythonConsoleLexer': ('pygments.lexers.python', 'Python console session', ('pycon',), (), ('text/x-python-doctest',)),
'PythonLexer': ('pygments.lexers.python', 'Python', ('python', 'py', 'sage'), ('*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript', '*.tac', '*.sage'), ('text/x-python', 'application/x-python')),
'PythonTracebackLexer': ('pygments.lexers.python', 'Python Traceback', ('pytb',), ('*.pytb',), ('text/x-python-traceback',)),
'QBasicLexer': ('pygments.lexers.basic', 'QBasic', ('qbasic', 'basic'), ('*.BAS', '*.bas'), ('text/basic',)),
'QVToLexer': ('pygments.lexers.qvt', 'QVTO', ('qvto', 'qvt'), ('*.qvto',), ()),
'QmlLexer': ('pygments.lexers.webmisc', 'QML', ('qml', 'qbs'), ('*.qml', '*.qbs'), ('application/x-qml', 'application/x-qt.qbs+qml')),
'RConsoleLexer': ('pygments.lexers.r', 'RConsole', ('rconsole', 'rout'), ('*.Rout',), ()),
'RNCCompactLexer': ('pygments.lexers.rnc', 'Relax-NG Compact', ('rnc', 'rng-compact'), ('*.rnc',), ()),
'RPMSpecLexer': ('pygments.lexers.installers', 'RPMSpec', ('spec',), ('*.spec',), ('text/x-rpm-spec',)),
'RacketLexer': ('pygments.lexers.lisp', 'Racket', ('racket', 'rkt'), ('*.rkt', '*.rktd', '*.rktl'), ('text/x-racket', 'application/x-racket')),
'RagelCLexer': ('pygments.lexers.parsers', 'Ragel in C Host', ('ragel-c',), ('*.rl',), ()),
'RagelCppLexer': ('pygments.lexers.parsers', 'Ragel in CPP Host', ('ragel-cpp',), ('*.rl',), ()),
'RagelDLexer': ('pygments.lexers.parsers', 'Ragel in D Host', ('ragel-d',), ('*.rl',), ()),
'RagelEmbeddedLexer': ('pygments.lexers.parsers', 'Embedded Ragel', ('ragel-em',), ('*.rl',), ()),
'RagelJavaLexer': ('pygments.lexers.parsers', 'Ragel in Java Host', ('ragel-java',), ('*.rl',), ()),
'RagelLexer': ('pygments.lexers.parsers', 'Ragel', ('ragel',), (), ()),
'RagelObjectiveCLexer': ('pygments.lexers.parsers', 'Ragel in Objective C Host', ('ragel-objc',), ('*.rl',), ()),
'RagelRubyLexer': ('pygments.lexers.parsers', 'Ragel in Ruby Host', ('ragel-ruby', 'ragel-rb'), ('*.rl',), ()),
'RawTokenLexer': ('pygments.lexers.special', 'Raw token data', ('raw',), (), ('application/x-pygments-tokens',)),
'RdLexer': ('pygments.lexers.r', 'Rd', ('rd',), ('*.Rd',), ('text/x-r-doc',)),
'RebolLexer': ('pygments.lexers.rebol', 'REBOL', ('rebol',), ('*.r', '*.r3', '*.reb'), ('text/x-rebol',)),
'RedLexer': ('pygments.lexers.rebol', 'Red', ('red', 'red/system'), ('*.red', '*.reds'), ('text/x-red', 'text/x-red-system')),
'RedcodeLexer': ('pygments.lexers.esoteric', 'Redcode', ('redcode',), ('*.cw',), ()),
'RegeditLexer': ('pygments.lexers.configs', 'reg', ('registry',), ('*.reg',), ('text/x-windows-registry',)),
'ResourceLexer': ('pygments.lexers.resource', 'ResourceBundle', ('resource', 'resourcebundle'), ('*.txt',), ()),
'RexxLexer': ('pygments.lexers.scripting', 'Rexx', ('rexx', 'arexx'), ('*.rexx', '*.rex', '*.rx', '*.arexx'), ('text/x-rexx',)),
'RhtmlLexer': ('pygments.lexers.templates', 'RHTML', ('rhtml', 'html+erb', 'html+ruby'), ('*.rhtml',), ('text/html+ruby',)),
'RoboconfGraphLexer': ('pygments.lexers.roboconf', 'Roboconf Graph', ('roboconf-graph',), ('*.graph',), ()),
'RoboconfInstancesLexer': ('pygments.lexers.roboconf', 'Roboconf Instances', ('roboconf-instances',), ('*.instances',), ()),
'RobotFrameworkLexer': ('pygments.lexers.robotframework', 'RobotFramework', ('robotframework',), ('*.txt', '*.robot'), ('text/x-robotframework',)),
'RqlLexer': ('pygments.lexers.sql', 'RQL', ('rql',), ('*.rql',), ('text/x-rql',)),
'RslLexer': ('pygments.lexers.dsls', 'RSL', ('rsl',), ('*.rsl',), ('text/rsl',)),
'RstLexer': ('pygments.lexers.markup', 'reStructuredText', ('rst', 'rest', 'restructuredtext'), ('*.rst', '*.rest'), ('text/x-rst', 'text/prs.fallenstein.rst')),
'RtsLexer': ('pygments.lexers.trafficscript', 'TrafficScript', ('rts', 'trafficscript'), ('*.rts',), ()),
'RubyConsoleLexer': ('pygments.lexers.ruby', 'Ruby irb session', ('rbcon', 'irb'), (), ('text/x-ruby-shellsession',)),
'RubyLexer': ('pygments.lexers.ruby', 'Ruby', ('rb', 'ruby', 'duby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby', 'Gemfile'), ('text/x-ruby', 'application/x-ruby')),
'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust',), ('*.rs', '*.rs.in'), ('text/rust',)),
'SASLexer': ('pygments.lexers.sas', 'SAS', ('sas',), ('*.SAS', '*.sas'), ('text/x-sas', 'text/sas', 'application/x-sas')),
'SLexer': ('pygments.lexers.r', 'S', ('splus', 's', 'r'), ('*.S', '*.R', '.Rhistory', '.Rprofile', '.Renviron'), ('text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile')),
'SMLLexer': ('pygments.lexers.ml', 'Standard ML', ('sml',), ('*.sml', '*.sig', '*.fun'), ('text/x-standardml', 'application/x-standardml')),
'SassLexer': ('pygments.lexers.css', 'Sass', ('sass',), ('*.sass',), ('text/x-sass',)),
'ScalaLexer': ('pygments.lexers.jvm', 'Scala', ('scala',), ('*.scala',), ('text/x-scala',)),
'ScamlLexer': ('pygments.lexers.html', 'Scaml', ('scaml',), ('*.scaml',), ('text/x-scaml',)),
'SchemeLexer': ('pygments.lexers.lisp', 'Scheme', ('scheme', 'scm'), ('*.scm', '*.ss'), ('text/x-scheme', 'application/x-scheme')),
'ScilabLexer': ('pygments.lexers.matlab', 'Scilab', ('scilab',), ('*.sci', '*.sce', '*.tst'), ('text/scilab',)),
'ScssLexer': ('pygments.lexers.css', 'SCSS', ('scss',), ('*.scss',), ('text/x-scss',)),
'ShenLexer': ('pygments.lexers.lisp', 'Shen', ('shen',), ('*.shen',), ('text/x-shen', 'application/x-shen')),
'SilverLexer': ('pygments.lexers.verification', 'Silver', ('silver',), ('*.sil', '*.vpr'), ()),
'SlimLexer': ('pygments.lexers.webmisc', 'Slim', ('slim',), ('*.slim',), ('text/x-slim',)),
'SmaliLexer': ('pygments.lexers.dalvik', 'Smali', ('smali',), ('*.smali',), ('text/smali',)),
'SmalltalkLexer': ('pygments.lexers.smalltalk', 'Smalltalk', ('smalltalk', 'squeak', 'st'), ('*.st',), ('text/x-smalltalk',)),
'SmartyLexer': ('pygments.lexers.templates', 'Smarty', ('smarty',), ('*.tpl',), ('application/x-smarty',)),
'SnobolLexer': ('pygments.lexers.snobol', 'Snobol', ('snobol',), ('*.snobol',), ('text/x-snobol',)),
'SnowballLexer': ('pygments.lexers.dsls', 'Snowball', ('snowball',), ('*.sbl',), ()),
'SourcePawnLexer': ('pygments.lexers.pawn', 'SourcePawn', ('sp',), ('*.sp',), ('text/x-sourcepawn',)),
'SourcesListLexer': ('pygments.lexers.installers', 'Debian Sourcelist', ('sourceslist', 'sources.list', 'debsources'), ('sources.list',), ()),
'SparqlLexer': ('pygments.lexers.rdf', 'SPARQL', ('sparql',), ('*.rq', '*.sparql'), ('application/sparql-query',)),
'SqlLexer': ('pygments.lexers.sql', 'SQL', ('sql',), ('*.sql',), ('text/x-sql',)),
'SqliteConsoleLexer': ('pygments.lexers.sql', 'sqlite3con', ('sqlite3',), ('*.sqlite3-console',), ('text/x-sqlite3-console',)),
'SquidConfLexer': ('pygments.lexers.configs', 'SquidConf', ('squidconf', 'squid.conf', 'squid'), ('squid.conf',), ('text/x-squidconf',)),
'SspLexer': ('pygments.lexers.templates', 'Scalate Server Page', ('ssp',), ('*.ssp',), ('application/x-ssp',)),
'StanLexer': ('pygments.lexers.modeling', 'Stan', ('stan',), ('*.stan',), ()),
'StataLexer': ('pygments.lexers.stata', 'Stata', ('stata', 'do'), ('*.do', '*.ado'), ('text/x-stata', 'text/stata', 'application/x-stata')),
'SuperColliderLexer': ('pygments.lexers.supercollider', 'SuperCollider', ('sc', 'supercollider'), ('*.sc', '*.scd'), ('application/supercollider', 'text/supercollider')),
'SwiftLexer': ('pygments.lexers.objective', 'Swift', ('swift',), ('*.swift',), ('text/x-swift',)),
'SwigLexer': ('pygments.lexers.c_like', 'SWIG', ('swig',), ('*.swg', '*.i'), ('text/swig',)),
'SystemVerilogLexer': ('pygments.lexers.hdl', 'systemverilog', ('systemverilog', 'sv'), ('*.sv', '*.svh'), ('text/x-systemverilog',)),
'TAPLexer': ('pygments.lexers.testing', 'TAP', ('tap',), ('*.tap',), ()),
'Tads3Lexer': ('pygments.lexers.int_fiction', 'TADS 3', ('tads3',), ('*.t',), ()),
'TasmLexer': ('pygments.lexers.asm', 'TASM', ('tasm',), ('*.asm', '*.ASM', '*.tasm'), ('text/x-tasm',)),
'TclLexer': ('pygments.lexers.tcl', 'Tcl', ('tcl',), ('*.tcl', '*.rvt'), ('text/x-tcl', 'text/x-script.tcl', 'application/x-tcl')),
'TcshLexer': ('pygments.lexers.shell', 'Tcsh', ('tcsh', 'csh'), ('*.tcsh', '*.csh'), ('application/x-csh',)),
'TcshSessionLexer': ('pygments.lexers.shell', 'Tcsh Session', ('tcshcon',), (), ()),
'TeaTemplateLexer': ('pygments.lexers.templates', 'Tea', ('tea',), ('*.tea',), ('text/x-tea',)),
'TermcapLexer': ('pygments.lexers.configs', 'Termcap', ('termcap',), ('termcap', 'termcap.src'), ()),
'TerminfoLexer': ('pygments.lexers.configs', 'Terminfo', ('terminfo',), ('terminfo', 'terminfo.src'), ()),
'TerraformLexer': ('pygments.lexers.configs', 'Terraform', ('terraform', 'tf'), ('*.tf',), ('application/x-tf', 'application/x-terraform')),
'TexLexer': ('pygments.lexers.markup', 'TeX', ('tex', 'latex'), ('*.tex', '*.aux', '*.toc'), ('text/x-tex', 'text/x-latex')),
'TextLexer': ('pygments.lexers.special', 'Text only', ('text',), ('*.txt',), ('text/plain',)),
'ThriftLexer': ('pygments.lexers.dsls', 'Thrift', ('thrift',), ('*.thrift',), ('application/x-thrift',)),
'TodotxtLexer': ('pygments.lexers.textfmts', 'Todotxt', ('todotxt',), ('todo.txt', '*.todotxt'), ('text/x-todo',)),
'TransactSqlLexer': ('pygments.lexers.sql', 'Transact-SQL', ('tsql', 't-sql'), ('*.sql',), ('text/x-tsql',)),
'TreetopLexer': ('pygments.lexers.parsers', 'Treetop', ('treetop',), ('*.treetop', '*.tt'), ()),
'TurtleLexer': ('pygments.lexers.rdf', 'Turtle', ('turtle',), ('*.ttl',), ('text/turtle', 'application/x-turtle')),
'TwigHtmlLexer': ('pygments.lexers.templates', 'HTML+Twig', ('html+twig',), ('*.twig',), ('text/html+twig',)),
'TwigLexer': ('pygments.lexers.templates', 'Twig', ('twig',), (), ('application/x-twig',)),
'TypeScriptLexer': ('pygments.lexers.javascript', 'TypeScript', ('ts', 'typescript'), ('*.ts', '*.tsx'), ('text/x-typescript',)),
'TypoScriptCssDataLexer': ('pygments.lexers.typoscript', 'TypoScriptCssData', ('typoscriptcssdata',), (), ()),
'TypoScriptHtmlDataLexer': ('pygments.lexers.typoscript', 'TypoScriptHtmlData', ('typoscripthtmldata',), (), ()),
'TypoScriptLexer': ('pygments.lexers.typoscript', 'TypoScript', ('typoscript',), ('*.ts', '*.txt'), ('text/x-typoscript',)),
'UrbiscriptLexer': ('pygments.lexers.urbi', 'UrbiScript', ('urbiscript',), ('*.u',), ('application/x-urbiscript',)),
'VCLLexer': ('pygments.lexers.varnish', 'VCL', ('vcl',), ('*.vcl',), ('text/x-vclsrc',)),
'VCLSnippetLexer': ('pygments.lexers.varnish', 'VCLSnippets', ('vclsnippets', 'vclsnippet'), (), ('text/x-vclsnippet',)),
'VCTreeStatusLexer': ('pygments.lexers.console', 'VCTreeStatus', ('vctreestatus',), (), ()),
'VGLLexer': ('pygments.lexers.dsls', 'VGL', ('vgl',), ('*.rpf',), ()),
'ValaLexer': ('pygments.lexers.c_like', 'Vala', ('vala', 'vapi'), ('*.vala', '*.vapi'), ('text/x-vala',)),
'VbNetAspxLexer': ('pygments.lexers.dotnet', 'aspx-vb', ('aspx-vb',), ('*.aspx', '*.asax', '*.ascx', '*.ashx', '*.asmx', '*.axd'), ()),
'VbNetLexer': ('pygments.lexers.dotnet', 'VB.net', ('vb.net', 'vbnet'), ('*.vb', '*.bas'), ('text/x-vbnet', 'text/x-vba')),
'VelocityHtmlLexer': ('pygments.lexers.templates', 'HTML+Velocity', ('html+velocity',), (), ('text/html+velocity',)),
'VelocityLexer': ('pygments.lexers.templates', 'Velocity', ('velocity',), ('*.vm', '*.fhtml'), ()),
'VelocityXmlLexer': ('pygments.lexers.templates', 'XML+Velocity', ('xml+velocity',), (), ('application/xml+velocity',)),
'VerilogLexer': ('pygments.lexers.hdl', 'verilog', ('verilog', 'v'), ('*.v',), ('text/x-verilog',)),
'VhdlLexer': ('pygments.lexers.hdl', 'vhdl', ('vhdl',), ('*.vhdl', '*.vhd'), ('text/x-vhdl',)),
'VimLexer': ('pygments.lexers.textedit', 'VimL', ('vim',), ('*.vim', '.vimrc', '.exrc', '.gvimrc', '_vimrc', '_exrc', '_gvimrc', 'vimrc', 'gvimrc'), ('text/x-vim',)),
'WDiffLexer': ('pygments.lexers.diff', 'WDiff', ('wdiff',), ('*.wdiff',), ()),
'WhileyLexer': ('pygments.lexers.whiley', 'Whiley', ('whiley',), ('*.whiley',), ('text/x-whiley',)),
'X10Lexer': ('pygments.lexers.x10', 'X10', ('x10', 'xten'), ('*.x10',), ('text/x-x10',)),
'XQueryLexer': ('pygments.lexers.webmisc', 'XQuery', ('xquery', 'xqy', 'xq', 'xql', 'xqm'), ('*.xqy', '*.xquery', '*.xq', '*.xql', '*.xqm'), ('text/xquery', 'application/xquery')),
'XmlDjangoLexer': ('pygments.lexers.templates', 'XML+Django/Jinja', ('xml+django', 'xml+jinja'), (), ('application/xml+django', 'application/xml+jinja')),
'XmlErbLexer': ('pygments.lexers.templates', 'XML+Ruby', ('xml+erb', 'xml+ruby'), (), ('application/xml+ruby',)),
'XmlLexer': ('pygments.lexers.html', 'XML', ('xml',), ('*.xml', '*.xsl', '*.rss', '*.xslt', '*.xsd', '*.wsdl', '*.wsf'), ('text/xml', 'application/xml', 'image/svg+xml', 'application/rss+xml', 'application/atom+xml')),
'XmlPhpLexer': ('pygments.lexers.templates', 'XML+PHP', ('xml+php',), (), ('application/xml+php',)),
'XmlSmartyLexer': ('pygments.lexers.templates', 'XML+Smarty', ('xml+smarty',), (), ('application/xml+smarty',)),
'XorgLexer': ('pygments.lexers.xorg', 'Xorg', ('xorg.conf',), ('xorg.conf',), ()),
'XsltLexer': ('pygments.lexers.html', 'XSLT', ('xslt',), ('*.xsl', '*.xslt', '*.xpl'), ('application/xsl+xml', 'application/xslt+xml')),
'XtendLexer': ('pygments.lexers.jvm', 'Xtend', ('xtend',), ('*.xtend',), ('text/x-xtend',)),
'XtlangLexer': ('pygments.lexers.lisp', 'xtlang', ('extempore',), ('*.xtm',), ()),
'YamlJinjaLexer': ('pygments.lexers.templates', 'YAML+Jinja', ('yaml+jinja', 'salt', 'sls'), ('*.sls',), ('text/x-yaml+jinja', 'text/x-sls')),
'YamlLexer': ('pygments.lexers.data', 'YAML', ('yaml',), ('*.yaml', '*.yml'), ('text/x-yaml',)),
'ZephirLexer': ('pygments.lexers.php', 'Zephir', ('zephir',), ('*.zep',), ()),
}
if __name__ == '__main__': # pragma: no cover
import sys
import os
# lookup lexers
found_lexers = []
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
for root, dirs, files in os.walk('.'):
for filename in files:
if filename.endswith('.py') and not filename.startswith('_'):
module_name = 'pygments.lexers%s.%s' % (
root[1:].replace('/', '.'), filename[:-3])
print(module_name)
module = __import__(module_name, None, None, [''])
for lexer_name in module.__all__:
lexer = getattr(module, lexer_name)
found_lexers.append(
'%r: %r' % (lexer_name,
(module_name,
lexer.name,
tuple(lexer.aliases),
tuple(lexer.filenames),
tuple(lexer.mimetypes))))
# sort them to make the diff minimal
found_lexers.sort()
# extract useful sourcecode from this file
with open(__file__) as fp:
content = fp.read()
# replace crnl to nl for Windows.
#
# Note that, originally, contributers should keep nl of master
# repository, for example by using some kind of automatic
# management EOL, like `EolExtension
# <https://www.mercurial-scm.org/wiki/EolExtension>`.
content = content.replace("\r\n", "\n")
header = content[:content.find('LEXERS = {')]
footer = content[content.find("if __name__ == '__main__':"):]
# write new file
with open(__file__, 'w') as fp:
fp.write(header)
fp.write('LEXERS = {\n %s,\n}\n\n' % ',\n '.join(found_lexers))
fp.write(footer)
print ('=== %d lexers processed.' % len(found_lexers))

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1,621 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexers._postgres_builtins
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Self-updating data files for PostgreSQL lexer.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
# Autogenerated: please edit them if you like wasting your time.
KEYWORDS = (
'ABORT',
'ABSOLUTE',
'ACCESS',
'ACTION',
'ADD',
'ADMIN',
'AFTER',
'AGGREGATE',
'ALL',
'ALSO',
'ALTER',
'ALWAYS',
'ANALYSE',
'ANALYZE',
'AND',
'ANY',
'ARRAY',
'AS',
'ASC',
'ASSERTION',
'ASSIGNMENT',
'ASYMMETRIC',
'AT',
'ATTRIBUTE',
'AUTHORIZATION',
'BACKWARD',
'BEFORE',
'BEGIN',
'BETWEEN',
'BIGINT',
'BINARY',
'BIT',
'BOOLEAN',
'BOTH',
'BY',
'CACHE',
'CALLED',
'CASCADE',
'CASCADED',
'CASE',
'CAST',
'CATALOG',
'CHAIN',
'CHAR',
'CHARACTER',
'CHARACTERISTICS',
'CHECK',
'CHECKPOINT',
'CLASS',
'CLOSE',
'CLUSTER',
'COALESCE',
'COLLATE',
'COLLATION',
'COLUMN',
'COMMENT',
'COMMENTS',
'COMMIT',
'COMMITTED',
'CONCURRENTLY',
'CONFIGURATION',
'CONNECTION',
'CONSTRAINT',
'CONSTRAINTS',
'CONTENT',
'CONTINUE',
'CONVERSION',
'COPY',
'COST',
'CREATE',
'CROSS',
'CSV',
'CURRENT',
'CURRENT_CATALOG',
'CURRENT_DATE',
'CURRENT_ROLE',
'CURRENT_SCHEMA',
'CURRENT_TIME',
'CURRENT_TIMESTAMP',
'CURRENT_USER',
'CURSOR',
'CYCLE',
'DATA',
'DATABASE',
'DAY',
'DEALLOCATE',
'DEC',
'DECIMAL',
'DECLARE',
'DEFAULT',
'DEFAULTS',
'DEFERRABLE',
'DEFERRED',
'DEFINER',
'DELETE',
'DELIMITER',
'DELIMITERS',
'DESC',
'DICTIONARY',
'DISABLE',
'DISCARD',
'DISTINCT',
'DO',
'DOCUMENT',
'DOMAIN',
'DOUBLE',
'DROP',
'EACH',
'ELSE',
'ENABLE',
'ENCODING',
'ENCRYPTED',
'END',
'ENUM',
'ESCAPE',
'EVENT',
'EXCEPT',
'EXCLUDE',
'EXCLUDING',
'EXCLUSIVE',
'EXECUTE',
'EXISTS',
'EXPLAIN',
'EXTENSION',
'EXTERNAL',
'EXTRACT',
'FALSE',
'FAMILY',
'FETCH',
'FILTER',
'FIRST',
'FLOAT',
'FOLLOWING',
'FOR',
'FORCE',
'FOREIGN',
'FORWARD',
'FREEZE',
'FROM',
'FULL',
'FUNCTION',
'FUNCTIONS',
'GLOBAL',
'GRANT',
'GRANTED',
'GREATEST',
'GROUP',
'HANDLER',
'HAVING',
'HEADER',
'HOLD',
'HOUR',
'IDENTITY',
'IF',
'ILIKE',
'IMMEDIATE',
'IMMUTABLE',
'IMPLICIT',
'IN',
'INCLUDING',
'INCREMENT',
'INDEX',
'INDEXES',
'INHERIT',
'INHERITS',
'INITIALLY',
'INLINE',
'INNER',
'INOUT',
'INPUT',
'INSENSITIVE',
'INSERT',
'INSTEAD',
'INT',
'INTEGER',
'INTERSECT',
'INTERVAL',
'INTO',
'INVOKER',
'IS',
'ISNULL',
'ISOLATION',
'JOIN',
'KEY',
'LABEL',
'LANGUAGE',
'LARGE',
'LAST',
'LATERAL',
'LC_COLLATE',
'LC_CTYPE',
'LEADING',
'LEAKPROOF',
'LEAST',
'LEFT',
'LEVEL',
'LIKE',
'LIMIT',
'LISTEN',
'LOAD',
'LOCAL',
'LOCALTIME',
'LOCALTIMESTAMP',
'LOCATION',
'LOCK',
'MAPPING',
'MATCH',
'MATERIALIZED',
'MAXVALUE',
'MINUTE',
'MINVALUE',
'MODE',
'MONTH',
'MOVE',
'NAME',
'NAMES',
'NATIONAL',
'NATURAL',
'NCHAR',
'NEXT',
'NO',
'NONE',
'NOT',
'NOTHING',
'NOTIFY',
'NOTNULL',
'NOWAIT',
'NULL',
'NULLIF',
'NULLS',
'NUMERIC',
'OBJECT',
'OF',
'OFF',
'OFFSET',
'OIDS',
'ON',
'ONLY',
'OPERATOR',
'OPTION',
'OPTIONS',
'OR',
'ORDER',
'ORDINALITY',
'OUT',
'OUTER',
'OVER',
'OVERLAPS',
'OVERLAY',
'OWNED',
'OWNER',
'PARSER',
'PARTIAL',
'PARTITION',
'PASSING',
'PASSWORD',
'PLACING',
'PLANS',
'POLICY',
'POSITION',
'PRECEDING',
'PRECISION',
'PREPARE',
'PREPARED',
'PRESERVE',
'PRIMARY',
'PRIOR',
'PRIVILEGES',
'PROCEDURAL',
'PROCEDURE',
'PROGRAM',
'QUOTE',
'RANGE',
'READ',
'REAL',
'REASSIGN',
'RECHECK',
'RECURSIVE',
'REF',
'REFERENCES',
'REFRESH',
'REINDEX',
'RELATIVE',
'RELEASE',
'RENAME',
'REPEATABLE',
'REPLACE',
'REPLICA',
'RESET',
'RESTART',
'RESTRICT',
'RETURNING',
'RETURNS',
'REVOKE',
'RIGHT',
'ROLE',
'ROLLBACK',
'ROW',
'ROWS',
'RULE',
'SAVEPOINT',
'SCHEMA',
'SCROLL',
'SEARCH',
'SECOND',
'SECURITY',
'SELECT',
'SEQUENCE',
'SEQUENCES',
'SERIALIZABLE',
'SERVER',
'SESSION',
'SESSION_USER',
'SET',
'SETOF',
'SHARE',
'SHOW',
'SIMILAR',
'SIMPLE',
'SMALLINT',
'SNAPSHOT',
'SOME',
'STABLE',
'STANDALONE',
'START',
'STATEMENT',
'STATISTICS',
'STDIN',
'STDOUT',
'STORAGE',
'STRICT',
'STRIP',
'SUBSTRING',
'SYMMETRIC',
'SYSID',
'SYSTEM',
'TABLE',
'TABLES',
'TABLESPACE',
'TEMP',
'TEMPLATE',
'TEMPORARY',
'TEXT',
'THEN',
'TIME',
'TIMESTAMP',
'TO',
'TRAILING',
'TRANSACTION',
'TREAT',
'TRIGGER',
'TRIM',
'TRUE',
'TRUNCATE',
'TRUSTED',
'TYPE',
'TYPES',
'UNBOUNDED',
'UNCOMMITTED',
'UNENCRYPTED',
'UNION',
'UNIQUE',
'UNKNOWN',
'UNLISTEN',
'UNLOGGED',
'UNTIL',
'UPDATE',
'USER',
'USING',
'VACUUM',
'VALID',
'VALIDATE',
'VALIDATOR',
'VALUE',
'VALUES',
'VARCHAR',
'VARIADIC',
'VARYING',
'VERBOSE',
'VERSION',
'VIEW',
'VIEWS',
'VOLATILE',
'WHEN',
'WHERE',
'WHITESPACE',
'WINDOW',
'WITH',
'WITHIN',
'WITHOUT',
'WORK',
'WRAPPER',
'WRITE',
'XML',
'XMLATTRIBUTES',
'XMLCONCAT',
'XMLELEMENT',
'XMLEXISTS',
'XMLFOREST',
'XMLPARSE',
'XMLPI',
'XMLROOT',
'XMLSERIALIZE',
'YEAR',
'YES',
'ZONE',
)
DATATYPES = (
'bigint',
'bigserial',
'bit',
'bit varying',
'bool',
'boolean',
'box',
'bytea',
'char',
'character',
'character varying',
'cidr',
'circle',
'date',
'decimal',
'double precision',
'float4',
'float8',
'inet',
'int',
'int2',
'int4',
'int8',
'integer',
'interval',
'json',
'jsonb',
'line',
'lseg',
'macaddr',
'money',
'numeric',
'path',
'pg_lsn',
'point',
'polygon',
'real',
'serial',
'serial2',
'serial4',
'serial8',
'smallint',
'smallserial',
'text',
'time',
'timestamp',
'timestamptz',
'timetz',
'tsquery',
'tsvector',
'txid_snapshot',
'uuid',
'varbit',
'varchar',
'with time zone',
'without time zone',
'xml',
)
PSEUDO_TYPES = (
'any',
'anyelement',
'anyarray',
'anynonarray',
'anyenum',
'anyrange',
'cstring',
'internal',
'language_handler',
'fdw_handler',
'record',
'trigger',
'void',
'opaque',
)
# Remove 'trigger' from types
PSEUDO_TYPES = tuple(sorted(set(PSEUDO_TYPES) - set(map(str.lower, KEYWORDS))))
PLPGSQL_KEYWORDS = (
'ALIAS', 'CONSTANT', 'DIAGNOSTICS', 'ELSIF', 'EXCEPTION', 'EXIT',
'FOREACH', 'GET', 'LOOP', 'NOTICE', 'OPEN', 'PERFORM', 'QUERY', 'RAISE',
'RETURN', 'REVERSE', 'SQLSTATE', 'WHILE',
)
if __name__ == '__main__': # pragma: no cover
import re
try:
from urllib import urlopen
except ImportError:
from urllib.request import urlopen
from pygments.util import format_lines
# One man's constant is another man's variable.
SOURCE_URL = 'https://github.com/postgres/postgres/raw/master'
KEYWORDS_URL = SOURCE_URL + '/doc/src/sgml/keywords.sgml'
DATATYPES_URL = SOURCE_URL + '/doc/src/sgml/datatype.sgml'
def update_myself():
data_file = list(urlopen(DATATYPES_URL))
datatypes = parse_datatypes(data_file)
pseudos = parse_pseudos(data_file)
keywords = parse_keywords(urlopen(KEYWORDS_URL))
update_consts(__file__, 'DATATYPES', datatypes)
update_consts(__file__, 'PSEUDO_TYPES', pseudos)
update_consts(__file__, 'KEYWORDS', keywords)
def parse_keywords(f):
kw = []
for m in re.finditer(
r'\s*<entry><token>([^<]+)</token></entry>\s*'
r'<entry>([^<]+)</entry>', f.read()):
kw.append(m.group(1))
if not kw:
raise ValueError('no keyword found')
kw.sort()
return kw
def parse_datatypes(f):
dt = set()
for line in f:
if '<sect1' in line:
break
if '<entry><type>' not in line:
continue
# Parse a string such as
# time [ (<replaceable>p</replaceable>) ] [ without time zone ]
# into types "time" and "without time zone"
# remove all the tags
line = re.sub("<replaceable>[^<]+</replaceable>", "", line)
line = re.sub("<[^>]+>", "", line)
# Drop the parts containing braces
for tmp in [t for tmp in line.split('[')
for t in tmp.split(']') if "(" not in t]:
for t in tmp.split(','):
t = t.strip()
if not t: continue
dt.add(" ".join(t.split()))
dt = list(dt)
dt.sort()
return dt
def parse_pseudos(f):
dt = []
re_start = re.compile(r'\s*<table id="datatype-pseudotypes-table">')
re_entry = re.compile(r'\s*<entry><type>([^<]+)</></entry>')
re_end = re.compile(r'\s*</table>')
f = iter(f)
for line in f:
if re_start.match(line) is not None:
break
else:
raise ValueError('pseudo datatypes table not found')
for line in f:
m = re_entry.match(line)
if m is not None:
dt.append(m.group(1))
if re_end.match(line) is not None:
break
else:
raise ValueError('end of pseudo datatypes table not found')
if not dt:
raise ValueError('pseudo datatypes not found')
return dt
def update_consts(filename, constname, content):
with open(filename) as f:
data = f.read()
# Line to start/end inserting
re_match = re.compile(r'^%s\s*=\s*\($.*?^\s*\)$' % constname, re.M | re.S)
m = re_match.search(data)
if not m:
raise ValueError('Could not find existing definition for %s' %
(constname,))
new_block = format_lines(constname, content)
data = data[:m.start()] + new_block + data[m.end():]
with open(filename, 'w') as f:
f.write(data)
update_myself()

File diff suppressed because it is too large Load Diff

View File

@ -1,532 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexers._stan_builtins
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
This file contains the names of functions for Stan used by
``pygments.lexers.math.StanLexer. This is for Stan language version 2.8.0.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
KEYWORDS = (
'else',
'for',
'if',
'in',
'increment_log_prob',
'integrate_ode',
'lp__',
'print',
'reject',
'return',
'while'
)
TYPES = (
'cholesky_factor_corr',
'cholesky_factor_cov',
'corr_matrix',
'cov_matrix',
'int',
'matrix',
'ordered',
'positive_ordered',
'real',
'row_vector',
'row_vectormatrix',
'simplex',
'unit_vector',
'vector',
'void')
FUNCTIONS = (
'Phi',
'Phi_approx',
'abs',
'acos',
'acosh',
'append_col',
'append_row',
'asin',
'asinh',
'atan',
'atan2',
'atanh',
'bernoulli_ccdf_log',
'bernoulli_cdf',
'bernoulli_cdf_log',
'bernoulli_log',
'bernoulli_logit_log',
'bernoulli_rng',
'bessel_first_kind',
'bessel_second_kind',
'beta_binomial_ccdf_log',
'beta_binomial_cdf',
'beta_binomial_cdf_log',
'beta_binomial_log',
'beta_binomial_rng',
'beta_ccdf_log',
'beta_cdf',
'beta_cdf_log',
'beta_log',
'beta_rng',
'binary_log_loss',
'binomial_ccdf_log',
'binomial_cdf',
'binomial_cdf_log',
'binomial_coefficient_log',
'binomial_log',
'binomial_logit_log',
'binomial_rng',
'block',
'categorical_log',
'categorical_logit_log',
'categorical_rng',
'cauchy_ccdf_log',
'cauchy_cdf',
'cauchy_cdf_log',
'cauchy_log',
'cauchy_rng',
'cbrt',
'ceil',
'chi_square_ccdf_log',
'chi_square_cdf',
'chi_square_cdf_log',
'chi_square_log',
'chi_square_rng',
'cholesky_decompose',
'col',
'cols',
'columns_dot_product',
'columns_dot_self',
'cos',
'cosh',
'crossprod',
'csr_extract_u',
'csr_extract_v',
'csr_extract_w',
'csr_matrix_times_vector',
'csr_to_dense_matrix',
'cumulative_sum',
'determinant',
'diag_matrix',
'diag_post_multiply',
'diag_pre_multiply',
'diagonal',
'digamma',
'dims',
'dirichlet_log',
'dirichlet_rng',
'distance',
'dot_product',
'dot_self',
'double_exponential_ccdf_log',
'double_exponential_cdf',
'double_exponential_cdf_log',
'double_exponential_log',
'double_exponential_rng',
'e',
'eigenvalues_sym',
'eigenvectors_sym',
'erf',
'erfc',
'exp',
'exp2',
'exp_mod_normal_ccdf_log',
'exp_mod_normal_cdf',
'exp_mod_normal_cdf_log',
'exp_mod_normal_log',
'exp_mod_normal_rng',
'expm1',
'exponential_ccdf_log',
'exponential_cdf',
'exponential_cdf_log',
'exponential_log',
'exponential_rng',
'fabs',
'falling_factorial',
'fdim',
'floor',
'fma',
'fmax',
'fmin',
'fmod',
'frechet_ccdf_log',
'frechet_cdf',
'frechet_cdf_log',
'frechet_log',
'frechet_rng',
'gamma_ccdf_log',
'gamma_cdf',
'gamma_cdf_log',
'gamma_log',
'gamma_p',
'gamma_q',
'gamma_rng',
'gaussian_dlm_obs_log',
'get_lp',
'gumbel_ccdf_log',
'gumbel_cdf',
'gumbel_cdf_log',
'gumbel_log',
'gumbel_rng',
'head',
'hypergeometric_log',
'hypergeometric_rng',
'hypot',
'if_else',
'int_step',
'inv',
'inv_chi_square_ccdf_log',
'inv_chi_square_cdf',
'inv_chi_square_cdf_log',
'inv_chi_square_log',
'inv_chi_square_rng',
'inv_cloglog',
'inv_gamma_ccdf_log',
'inv_gamma_cdf',
'inv_gamma_cdf_log',
'inv_gamma_log',
'inv_gamma_rng',
'inv_logit',
'inv_phi',
'inv_sqrt',
'inv_square',
'inv_wishart_log',
'inv_wishart_rng',
'inverse',
'inverse_spd',
'is_inf',
'is_nan',
'lbeta',
'lgamma',
'lkj_corr_cholesky_log',
'lkj_corr_cholesky_rng',
'lkj_corr_log',
'lkj_corr_rng',
'lmgamma',
'log',
'log10',
'log1m',
'log1m_exp',
'log1m_inv_logit',
'log1p',
'log1p_exp',
'log2',
'log_determinant',
'log_diff_exp',
'log_falling_factorial',
'log_inv_logit',
'log_mix',
'log_rising_factorial',
'log_softmax',
'log_sum_exp',
'logistic_ccdf_log',
'logistic_cdf',
'logistic_cdf_log',
'logistic_log',
'logistic_rng',
'logit',
'lognormal_ccdf_log',
'lognormal_cdf',
'lognormal_cdf_log',
'lognormal_log',
'lognormal_rng',
'machine_precision',
'max',
'mdivide_left_tri_low',
'mdivide_right_tri_low',
'mean',
'min',
'modified_bessel_first_kind',
'modified_bessel_second_kind',
'multi_gp_cholesky_log',
'multi_gp_log',
'multi_normal_cholesky_log',
'multi_normal_cholesky_rng',
'multi_normal_log',
'multi_normal_prec_log',
'multi_normal_rng',
'multi_student_t_log',
'multi_student_t_rng',
'multinomial_log',
'multinomial_rng',
'multiply_log',
'multiply_lower_tri_self_transpose',
'neg_binomial_2_ccdf_log',
'neg_binomial_2_cdf',
'neg_binomial_2_cdf_log',
'neg_binomial_2_log',
'neg_binomial_2_log_log',
'neg_binomial_2_log_rng',
'neg_binomial_2_rng',
'neg_binomial_ccdf_log',
'neg_binomial_cdf',
'neg_binomial_cdf_log',
'neg_binomial_log',
'neg_binomial_rng',
'negative_infinity',
'normal_ccdf_log',
'normal_cdf',
'normal_cdf_log',
'normal_log',
'normal_rng',
'not_a_number',
'num_elements',
'ordered_logistic_log',
'ordered_logistic_rng',
'owens_t',
'pareto_ccdf_log',
'pareto_cdf',
'pareto_cdf_log',
'pareto_log',
'pareto_rng',
'pareto_type_2_ccdf_log',
'pareto_type_2_cdf',
'pareto_type_2_cdf_log',
'pareto_type_2_log',
'pareto_type_2_rng',
'pi',
'poisson_ccdf_log',
'poisson_cdf',
'poisson_cdf_log',
'poisson_log',
'poisson_log_log',
'poisson_log_rng',
'poisson_rng',
'positive_infinity',
'pow',
'prod',
'qr_Q',
'qr_R',
'quad_form',
'quad_form_diag',
'quad_form_sym',
'rank',
'rayleigh_ccdf_log',
'rayleigh_cdf',
'rayleigh_cdf_log',
'rayleigh_log',
'rayleigh_rng',
'rep_array',
'rep_matrix',
'rep_row_vector',
'rep_vector',
'rising_factorial',
'round',
'row',
'rows',
'rows_dot_product',
'rows_dot_self',
'scaled_inv_chi_square_ccdf_log',
'scaled_inv_chi_square_cdf',
'scaled_inv_chi_square_cdf_log',
'scaled_inv_chi_square_log',
'scaled_inv_chi_square_rng',
'sd',
'segment',
'sin',
'singular_values',
'sinh',
'size',
'skew_normal_ccdf_log',
'skew_normal_cdf',
'skew_normal_cdf_log',
'skew_normal_log',
'skew_normal_rng',
'softmax',
'sort_asc',
'sort_desc',
'sort_indices_asc',
'sort_indices_desc',
'sqrt',
'sqrt2',
'square',
'squared_distance',
'step',
'student_t_ccdf_log',
'student_t_cdf',
'student_t_cdf_log',
'student_t_log',
'student_t_rng',
'sub_col',
'sub_row',
'sum',
'tail',
'tan',
'tanh',
'tcrossprod',
'tgamma',
'to_array_1d',
'to_array_2d',
'to_matrix',
'to_row_vector',
'to_vector',
'trace',
'trace_gen_quad_form',
'trace_quad_form',
'trigamma',
'trunc',
'uniform_ccdf_log',
'uniform_cdf',
'uniform_cdf_log',
'uniform_log',
'uniform_rng',
'variance',
'von_mises_log',
'von_mises_rng',
'weibull_ccdf_log',
'weibull_cdf',
'weibull_cdf_log',
'weibull_log',
'weibull_rng',
'wiener_log',
'wishart_log',
'wishart_rng'
)
DISTRIBUTIONS = (
'bernoulli',
'bernoulli_logit',
'beta',
'beta_binomial',
'binomial',
'binomial_logit',
'categorical',
'categorical_logit',
'cauchy',
'chi_square',
'dirichlet',
'double_exponential',
'exp_mod_normal',
'exponential',
'frechet',
'gamma',
'gaussian_dlm_obs',
'gumbel',
'hypergeometric',
'inv_chi_square',
'inv_gamma',
'inv_wishart',
'lkj_corr',
'lkj_corr_cholesky',
'logistic',
'lognormal',
'multi_gp',
'multi_gp_cholesky',
'multi_normal',
'multi_normal_cholesky',
'multi_normal_prec',
'multi_student_t',
'multinomial',
'neg_binomial',
'neg_binomial_2',
'neg_binomial_2_log',
'normal',
'ordered_logistic',
'pareto',
'pareto_type_2',
'poisson',
'poisson_log',
'rayleigh',
'scaled_inv_chi_square',
'skew_normal',
'student_t',
'uniform',
'von_mises',
'weibull',
'wiener',
'wishart'
)
RESERVED = (
'alignas',
'alignof',
'and',
'and_eq',
'asm',
'auto',
'bitand',
'bitor',
'bool',
'break',
'case',
'catch',
'char',
'char16_t',
'char32_t',
'class',
'compl',
'const',
'const_cast',
'constexpr',
'continue',
'decltype',
'default',
'delete',
'do',
'double',
'dynamic_cast',
'enum',
'explicit',
'export',
'extern',
'false',
'false',
'float',
'friend',
'fvar',
'goto',
'inline',
'int',
'long',
'mutable',
'namespace',
'new',
'noexcept',
'not',
'not_eq',
'nullptr',
'operator',
'or',
'or_eq',
'private',
'protected',
'public',
'register',
'reinterpret_cast',
'repeat',
'short',
'signed',
'sizeof',
'static',
'static_assert',
'static_cast',
'struct',
'switch',
'template',
'then',
'this',
'thread_local',
'throw',
'true',
'true',
'try',
'typedef',
'typeid',
'typename',
'union',
'unsigned',
'until',
'using',
'var',
'virtual',
'void',
'volatile',
'wchar_t',
'xor',
'xor_eq'
)

View File

@ -1,419 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexers._stata_builtins
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Builtins for Stata
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
builtins_base = (
"if", "else", "in", "foreach", "for", "forv", "forva",
"forval", "forvalu", "forvalue", "forvalues", "by", "bys",
"bysort", "quietly", "qui", "about", "ac",
"ac_7", "acprplot", "acprplot_7", "adjust", "ado", "adopath",
"adoupdate", "alpha", "ameans", "an", "ano", "anov", "anova",
"anova_estat", "anova_terms", "anovadef", "aorder", "ap", "app",
"appe", "appen", "append", "arch", "arch_dr", "arch_estat",
"arch_p", "archlm", "areg", "areg_p", "args", "arima",
"arima_dr", "arima_estat", "arima_p", "as", "asmprobit",
"asmprobit_estat", "asmprobit_lf", "asmprobit_mfx__dlg",
"asmprobit_p", "ass", "asse", "asser", "assert", "avplot",
"avplot_7", "avplots", "avplots_7", "bcskew0", "bgodfrey",
"binreg", "bip0_lf", "biplot", "bipp_lf", "bipr_lf",
"bipr_p", "biprobit", "bitest", "bitesti", "bitowt", "blogit",
"bmemsize", "boot", "bootsamp", "bootstrap", "bootstrap_8",
"boxco_l", "boxco_p", "boxcox", "boxcox_6", "boxcox_p",
"bprobit", "br", "break", "brier", "bro", "brow", "brows",
"browse", "brr", "brrstat", "bs", "bs_7", "bsampl_w",
"bsample", "bsample_7", "bsqreg", "bstat", "bstat_7", "bstat_8",
"bstrap", "bstrap_7", "ca", "ca_estat", "ca_p", "cabiplot",
"camat", "canon", "canon_8", "canon_8_p", "canon_estat",
"canon_p", "cap", "caprojection", "capt", "captu", "captur",
"capture", "cat", "cc", "cchart", "cchart_7", "cci",
"cd", "censobs_table", "centile", "cf", "char", "chdir",
"checkdlgfiles", "checkestimationsample", "checkhlpfiles",
"checksum", "chelp", "ci", "cii", "cl", "class", "classutil",
"clear", "cli", "clis", "clist", "clo", "clog", "clog_lf",
"clog_p", "clogi", "clogi_sw", "clogit", "clogit_lf",
"clogit_p", "clogitp", "clogl_sw", "cloglog", "clonevar",
"clslistarray", "cluster", "cluster_measures", "cluster_stop",
"cluster_tree", "cluster_tree_8", "clustermat", "cmdlog",
"cnr", "cnre", "cnreg", "cnreg_p", "cnreg_sw", "cnsreg",
"codebook", "collaps4", "collapse", "colormult_nb",
"colormult_nw", "compare", "compress", "conf", "confi",
"confir", "confirm", "conren", "cons", "const", "constr",
"constra", "constrai", "constrain", "constraint", "continue",
"contract", "copy", "copyright", "copysource", "cor", "corc",
"corr", "corr2data", "corr_anti", "corr_kmo", "corr_smc",
"corre", "correl", "correla", "correlat", "correlate",
"corrgram", "cou", "coun", "count", "cox", "cox_p", "cox_sw",
"coxbase", "coxhaz", "coxvar", "cprplot", "cprplot_7",
"crc", "cret", "cretu", "cretur", "creturn", "cross", "cs",
"cscript", "cscript_log", "csi", "ct", "ct_is", "ctset",
"ctst_5", "ctst_st", "cttost", "cumsp", "cumsp_7", "cumul",
"cusum", "cusum_7", "cutil", "d", "datasig", "datasign",
"datasigna", "datasignat", "datasignatu", "datasignatur",
"datasignature", "datetof", "db", "dbeta", "de", "dec",
"deco", "decod", "decode", "deff", "des", "desc", "descr",
"descri", "describ", "describe", "destring", "dfbeta",
"dfgls", "dfuller", "di", "di_g", "dir", "dirstats", "dis",
"discard", "disp", "disp_res", "disp_s", "displ", "displa",
"display", "distinct", "do", "doe", "doed", "doedi",
"doedit", "dotplot", "dotplot_7", "dprobit", "drawnorm",
"drop", "ds", "ds_util", "dstdize", "duplicates", "durbina",
"dwstat", "dydx", "e", "ed", "edi", "edit", "egen",
"eivreg", "emdef", "en", "enc", "enco", "encod", "encode",
"eq", "erase", "ereg", "ereg_lf", "ereg_p", "ereg_sw",
"ereghet", "ereghet_glf", "ereghet_glf_sh", "ereghet_gp",
"ereghet_ilf", "ereghet_ilf_sh", "ereghet_ip", "eret",
"eretu", "eretur", "ereturn", "err", "erro", "error", "est",
"est_cfexist", "est_cfname", "est_clickable", "est_expand",
"est_hold", "est_table", "est_unhold", "est_unholdok",
"estat", "estat_default", "estat_summ", "estat_vce_only",
"esti", "estimates", "etodow", "etof", "etomdy", "ex",
"exi", "exit", "expand", "expandcl", "fac", "fact", "facto",
"factor", "factor_estat", "factor_p", "factor_pca_rotated",
"factor_rotate", "factormat", "fcast", "fcast_compute",
"fcast_graph", "fdades", "fdadesc", "fdadescr", "fdadescri",
"fdadescrib", "fdadescribe", "fdasav", "fdasave", "fdause",
"fh_st", "open", "read", "close",
"file", "filefilter", "fillin", "find_hlp_file", "findfile",
"findit", "findit_7", "fit", "fl", "fli", "flis", "flist",
"for5_0", "form", "forma", "format", "fpredict", "frac_154",
"frac_adj", "frac_chk", "frac_cox", "frac_ddp", "frac_dis",
"frac_dv", "frac_in", "frac_mun", "frac_pp", "frac_pq",
"frac_pv", "frac_wgt", "frac_xo", "fracgen", "fracplot",
"fracplot_7", "fracpoly", "fracpred", "fron_ex", "fron_hn",
"fron_p", "fron_tn", "fron_tn2", "frontier", "ftodate", "ftoe",
"ftomdy", "ftowdate", "g", "gamhet_glf", "gamhet_gp",
"gamhet_ilf", "gamhet_ip", "gamma", "gamma_d2", "gamma_p",
"gamma_sw", "gammahet", "gdi_hexagon", "gdi_spokes", "ge",
"gen", "gene", "gener", "genera", "generat", "generate",
"genrank", "genstd", "genvmean", "gettoken", "gl", "gladder",
"gladder_7", "glim_l01", "glim_l02", "glim_l03", "glim_l04",
"glim_l05", "glim_l06", "glim_l07", "glim_l08", "glim_l09",
"glim_l10", "glim_l11", "glim_l12", "glim_lf", "glim_mu",
"glim_nw1", "glim_nw2", "glim_nw3", "glim_p", "glim_v1",
"glim_v2", "glim_v3", "glim_v4", "glim_v5", "glim_v6",
"glim_v7", "glm", "glm_6", "glm_p", "glm_sw", "glmpred", "glo",
"glob", "globa", "global", "glogit", "glogit_8", "glogit_p",
"gmeans", "gnbre_lf", "gnbreg", "gnbreg_5", "gnbreg_p",
"gomp_lf", "gompe_sw", "gomper_p", "gompertz", "gompertzhet",
"gomphet_glf", "gomphet_glf_sh", "gomphet_gp", "gomphet_ilf",
"gomphet_ilf_sh", "gomphet_ip", "gphdot", "gphpen",
"gphprint", "gprefs", "gprobi_p", "gprobit", "gprobit_8", "gr",
"gr7", "gr_copy", "gr_current", "gr_db", "gr_describe",
"gr_dir", "gr_draw", "gr_draw_replay", "gr_drop", "gr_edit",
"gr_editviewopts", "gr_example", "gr_example2", "gr_export",
"gr_print", "gr_qscheme", "gr_query", "gr_read", "gr_rename",
"gr_replay", "gr_save", "gr_set", "gr_setscheme", "gr_table",
"gr_undo", "gr_use", "graph", "graph7", "grebar", "greigen",
"greigen_7", "greigen_8", "grmeanby", "grmeanby_7",
"gs_fileinfo", "gs_filetype", "gs_graphinfo", "gs_stat",
"gsort", "gwood", "h", "hadimvo", "hareg", "hausman",
"haver", "he", "heck_d2", "heckma_p", "heckman", "heckp_lf",
"heckpr_p", "heckprob", "hel", "help", "hereg", "hetpr_lf",
"hetpr_p", "hetprob", "hettest", "hexdump", "hilite",
"hist", "hist_7", "histogram", "hlogit", "hlu", "hmeans",
"hotel", "hotelling", "hprobit", "hreg", "hsearch", "icd9",
"icd9_ff", "icd9p", "iis", "impute", "imtest", "inbase",
"include", "inf", "infi", "infil", "infile", "infix", "inp",
"inpu", "input", "ins", "insheet", "insp", "inspe",
"inspec", "inspect", "integ", "inten", "intreg", "intreg_7",
"intreg_p", "intrg2_ll", "intrg_ll", "intrg_ll2", "ipolate",
"iqreg", "ir", "irf", "irf_create", "irfm", "iri", "is_svy",
"is_svysum", "isid", "istdize", "ivprob_1_lf", "ivprob_lf",
"ivprobit", "ivprobit_p", "ivreg", "ivreg_footnote",
"ivtob_1_lf", "ivtob_lf", "ivtobit", "ivtobit_p", "jackknife",
"jacknife", "jknife", "jknife_6", "jknife_8", "jkstat",
"joinby", "kalarma1", "kap", "kap_3", "kapmeier", "kappa",
"kapwgt", "kdensity", "kdensity_7", "keep", "ksm", "ksmirnov",
"ktau", "kwallis", "l", "la", "lab", "labe", "label",
"labelbook", "ladder", "levels", "levelsof", "leverage",
"lfit", "lfit_p", "li", "lincom", "line", "linktest",
"lis", "list", "lloghet_glf", "lloghet_glf_sh", "lloghet_gp",
"lloghet_ilf", "lloghet_ilf_sh", "lloghet_ip", "llogi_sw",
"llogis_p", "llogist", "llogistic", "llogistichet",
"lnorm_lf", "lnorm_sw", "lnorma_p", "lnormal", "lnormalhet",
"lnormhet_glf", "lnormhet_glf_sh", "lnormhet_gp",
"lnormhet_ilf", "lnormhet_ilf_sh", "lnormhet_ip", "lnskew0",
"loadingplot", "loc", "loca", "local", "log", "logi",
"logis_lf", "logistic", "logistic_p", "logit", "logit_estat",
"logit_p", "loglogs", "logrank", "loneway", "lookfor",
"lookup", "lowess", "lowess_7", "lpredict", "lrecomp", "lroc",
"lroc_7", "lrtest", "ls", "lsens", "lsens_7", "lsens_x",
"lstat", "ltable", "ltable_7", "ltriang", "lv", "lvr2plot",
"lvr2plot_7", "m", "ma", "mac", "macr", "macro", "makecns",
"man", "manova", "manova_estat", "manova_p", "manovatest",
"mantel", "mark", "markin", "markout", "marksample", "mat",
"mat_capp", "mat_order", "mat_put_rr", "mat_rapp", "mata",
"mata_clear", "mata_describe", "mata_drop", "mata_matdescribe",
"mata_matsave", "mata_matuse", "mata_memory", "mata_mlib",
"mata_mosave", "mata_rename", "mata_which", "matalabel",
"matcproc", "matlist", "matname", "matr", "matri",
"matrix", "matrix_input__dlg", "matstrik", "mcc", "mcci",
"md0_", "md1_", "md1debug_", "md2_", "md2debug_", "mds",
"mds_estat", "mds_p", "mdsconfig", "mdslong", "mdsmat",
"mdsshepard", "mdytoe", "mdytof", "me_derd", "mean",
"means", "median", "memory", "memsize", "meqparse", "mer",
"merg", "merge", "mfp", "mfx", "mhelp", "mhodds", "minbound",
"mixed_ll", "mixed_ll_reparm", "mkassert", "mkdir",
"mkmat", "mkspline", "ml", "ml_5", "ml_adjs", "ml_bhhhs",
"ml_c_d", "ml_check", "ml_clear", "ml_cnt", "ml_debug",
"ml_defd", "ml_e0", "ml_e0_bfgs", "ml_e0_cycle", "ml_e0_dfp",
"ml_e0i", "ml_e1", "ml_e1_bfgs", "ml_e1_bhhh", "ml_e1_cycle",
"ml_e1_dfp", "ml_e2", "ml_e2_cycle", "ml_ebfg0", "ml_ebfr0",
"ml_ebfr1", "ml_ebh0q", "ml_ebhh0", "ml_ebhr0", "ml_ebr0i",
"ml_ecr0i", "ml_edfp0", "ml_edfr0", "ml_edfr1", "ml_edr0i",
"ml_eds", "ml_eer0i", "ml_egr0i", "ml_elf", "ml_elf_bfgs",
"ml_elf_bhhh", "ml_elf_cycle", "ml_elf_dfp", "ml_elfi",
"ml_elfs", "ml_enr0i", "ml_enrr0", "ml_erdu0", "ml_erdu0_bfgs",
"ml_erdu0_bhhh", "ml_erdu0_bhhhq", "ml_erdu0_cycle",
"ml_erdu0_dfp", "ml_erdu0_nrbfgs", "ml_exde", "ml_footnote",
"ml_geqnr", "ml_grad0", "ml_graph", "ml_hbhhh", "ml_hd0",
"ml_hold", "ml_init", "ml_inv", "ml_log", "ml_max",
"ml_mlout", "ml_mlout_8", "ml_model", "ml_nb0", "ml_opt",
"ml_p", "ml_plot", "ml_query", "ml_rdgrd", "ml_repor",
"ml_s_e", "ml_score", "ml_searc", "ml_technique", "ml_unhold",
"mleval", "mlf_", "mlmatbysum", "mlmatsum", "mlog", "mlogi",
"mlogit", "mlogit_footnote", "mlogit_p", "mlopts", "mlsum",
"mlvecsum", "mnl0_", "mor", "more", "mov", "move", "mprobit",
"mprobit_lf", "mprobit_p", "mrdu0_", "mrdu1_", "mvdecode",
"mvencode", "mvreg", "mvreg_estat", "n", "nbreg",
"nbreg_al", "nbreg_lf", "nbreg_p", "nbreg_sw", "nestreg", "net",
"newey", "newey_7", "newey_p", "news", "nl", "nl_7", "nl_9",
"nl_9_p", "nl_p", "nl_p_7", "nlcom", "nlcom_p", "nlexp2",
"nlexp2_7", "nlexp2a", "nlexp2a_7", "nlexp3", "nlexp3_7",
"nlgom3", "nlgom3_7", "nlgom4", "nlgom4_7", "nlinit", "nllog3",
"nllog3_7", "nllog4", "nllog4_7", "nlog_rd", "nlogit",
"nlogit_p", "nlogitgen", "nlogittree", "nlpred", "no",
"nobreak", "noi", "nois", "noisi", "noisil", "noisily", "note",
"notes", "notes_dlg", "nptrend", "numlabel", "numlist", "odbc",
"old_ver", "olo", "olog", "ologi", "ologi_sw", "ologit",
"ologit_p", "ologitp", "on", "one", "onew", "onewa", "oneway",
"op_colnm", "op_comp", "op_diff", "op_inv", "op_str", "opr",
"opro", "oprob", "oprob_sw", "oprobi", "oprobi_p", "oprobit",
"oprobitp", "opts_exclusive", "order", "orthog", "orthpoly",
"ou", "out", "outf", "outfi", "outfil", "outfile", "outs",
"outsh", "outshe", "outshee", "outsheet", "ovtest", "pac",
"pac_7", "palette", "parse", "parse_dissim", "pause", "pca",
"pca_8", "pca_display", "pca_estat", "pca_p", "pca_rotate",
"pcamat", "pchart", "pchart_7", "pchi", "pchi_7", "pcorr",
"pctile", "pentium", "pergram", "pergram_7", "permute",
"permute_8", "personal", "peto_st", "pkcollapse", "pkcross",
"pkequiv", "pkexamine", "pkexamine_7", "pkshape", "pksumm",
"pksumm_7", "pl", "plo", "plot", "plugin", "pnorm",
"pnorm_7", "poisgof", "poiss_lf", "poiss_sw", "poisso_p",
"poisson", "poisson_estat", "post", "postclose", "postfile",
"postutil", "pperron", "pr", "prais", "prais_e", "prais_e2",
"prais_p", "predict", "predictnl", "preserve", "print",
"pro", "prob", "probi", "probit", "probit_estat", "probit_p",
"proc_time", "procoverlay", "procrustes", "procrustes_estat",
"procrustes_p", "profiler", "prog", "progr", "progra",
"program", "prop", "proportion", "prtest", "prtesti", "pwcorr",
"pwd", "q", "s", "qby", "qbys", "qchi", "qchi_7", "qladder",
"qladder_7", "qnorm", "qnorm_7", "qqplot", "qqplot_7", "qreg",
"qreg_c", "qreg_p", "qreg_sw", "qu", "quadchk", "quantile",
"quantile_7", "que", "quer", "query", "range", "ranksum",
"ratio", "rchart", "rchart_7", "rcof", "recast", "reclink",
"recode", "reg", "reg3", "reg3_p", "regdw", "regr", "regre",
"regre_p2", "regres", "regres_p", "regress", "regress_estat",
"regriv_p", "remap", "ren", "rena", "renam", "rename",
"renpfix", "repeat", "replace", "report", "reshape",
"restore", "ret", "retu", "retur", "return", "rm", "rmdir",
"robvar", "roccomp", "roccomp_7", "roccomp_8", "rocf_lf",
"rocfit", "rocfit_8", "rocgold", "rocplot", "rocplot_7",
"roctab", "roctab_7", "rolling", "rologit", "rologit_p",
"rot", "rota", "rotat", "rotate", "rotatemat", "rreg",
"rreg_p", "ru", "run", "runtest", "rvfplot", "rvfplot_7",
"rvpplot", "rvpplot_7", "sa", "safesum", "sample",
"sampsi", "sav", "save", "savedresults", "saveold", "sc",
"sca", "scal", "scala", "scalar", "scatter", "scm_mine",
"sco", "scob_lf", "scob_p", "scobi_sw", "scobit", "scor",
"score", "scoreplot", "scoreplot_help", "scree", "screeplot",
"screeplot_help", "sdtest", "sdtesti", "se", "search",
"separate", "seperate", "serrbar", "serrbar_7", "serset", "set",
"set_defaults", "sfrancia", "sh", "she", "shel", "shell",
"shewhart", "shewhart_7", "signestimationsample", "signrank",
"signtest", "simul", "simul_7", "simulate", "simulate_8",
"sktest", "sleep", "slogit", "slogit_d2", "slogit_p", "smooth",
"snapspan", "so", "sor", "sort", "spearman", "spikeplot",
"spikeplot_7", "spikeplt", "spline_x", "split", "sqreg",
"sqreg_p", "sret", "sretu", "sretur", "sreturn", "ssc", "st",
"st_ct", "st_hc", "st_hcd", "st_hcd_sh", "st_is", "st_issys",
"st_note", "st_promo", "st_set", "st_show", "st_smpl",
"st_subid", "stack", "statsby", "statsby_8", "stbase", "stci",
"stci_7", "stcox", "stcox_estat", "stcox_fr", "stcox_fr_ll",
"stcox_p", "stcox_sw", "stcoxkm", "stcoxkm_7", "stcstat",
"stcurv", "stcurve", "stcurve_7", "stdes", "stem", "stepwise",
"stereg", "stfill", "stgen", "stir", "stjoin", "stmc", "stmh",
"stphplot", "stphplot_7", "stphtest", "stphtest_7",
"stptime", "strate", "strate_7", "streg", "streg_sw", "streset",
"sts", "sts_7", "stset", "stsplit", "stsum", "sttocc",
"sttoct", "stvary", "stweib", "su", "suest", "suest_8",
"sum", "summ", "summa", "summar", "summari", "summariz",
"summarize", "sunflower", "sureg", "survcurv", "survsum",
"svar", "svar_p", "svmat", "svy", "svy_disp", "svy_dreg",
"svy_est", "svy_est_7", "svy_estat", "svy_get", "svy_gnbreg_p",
"svy_head", "svy_header", "svy_heckman_p", "svy_heckprob_p",
"svy_intreg_p", "svy_ivreg_p", "svy_logistic_p", "svy_logit_p",
"svy_mlogit_p", "svy_nbreg_p", "svy_ologit_p", "svy_oprobit_p",
"svy_poisson_p", "svy_probit_p", "svy_regress_p", "svy_sub",
"svy_sub_7", "svy_x", "svy_x_7", "svy_x_p", "svydes",
"svydes_8", "svygen", "svygnbreg", "svyheckman", "svyheckprob",
"svyintreg", "svyintreg_7", "svyintrg", "svyivreg", "svylc",
"svylog_p", "svylogit", "svymarkout", "svymarkout_8",
"svymean", "svymlog", "svymlogit", "svynbreg", "svyolog",
"svyologit", "svyoprob", "svyoprobit", "svyopts",
"svypois", "svypois_7", "svypoisson", "svyprobit", "svyprobt",
"svyprop", "svyprop_7", "svyratio", "svyreg", "svyreg_p",
"svyregress", "svyset", "svyset_7", "svyset_8", "svytab",
"svytab_7", "svytest", "svytotal", "sw", "sw_8", "swcnreg",
"swcox", "swereg", "swilk", "swlogis", "swlogit",
"swologit", "swoprbt", "swpois", "swprobit", "swqreg",
"swtobit", "swweib", "symmetry", "symmi", "symplot",
"symplot_7", "syntax", "sysdescribe", "sysdir", "sysuse",
"szroeter", "ta", "tab", "tab1", "tab2", "tab_or", "tabd",
"tabdi", "tabdis", "tabdisp", "tabi", "table", "tabodds",
"tabodds_7", "tabstat", "tabu", "tabul", "tabula", "tabulat",
"tabulate", "te", "tempfile", "tempname", "tempvar", "tes",
"test", "testnl", "testparm", "teststd", "tetrachoric",
"time_it", "timer", "tis", "tob", "tobi", "tobit", "tobit_p",
"tobit_sw", "token", "tokeni", "tokeniz", "tokenize",
"tostring", "total", "translate", "translator", "transmap",
"treat_ll", "treatr_p", "treatreg", "trim", "trnb_cons",
"trnb_mean", "trpoiss_d2", "trunc_ll", "truncr_p", "truncreg",
"tsappend", "tset", "tsfill", "tsline", "tsline_ex",
"tsreport", "tsrevar", "tsrline", "tsset", "tssmooth",
"tsunab", "ttest", "ttesti", "tut_chk", "tut_wait", "tutorial",
"tw", "tware_st", "two", "twoway", "twoway__fpfit_serset",
"twoway__function_gen", "twoway__histogram_gen",
"twoway__ipoint_serset", "twoway__ipoints_serset",
"twoway__kdensity_gen", "twoway__lfit_serset",
"twoway__normgen_gen", "twoway__pci_serset",
"twoway__qfit_serset", "twoway__scatteri_serset",
"twoway__sunflower_gen", "twoway_ksm_serset", "ty", "typ",
"type", "typeof", "u", "unab", "unabbrev", "unabcmd",
"update", "us", "use", "uselabel", "var", "var_mkcompanion",
"var_p", "varbasic", "varfcast", "vargranger", "varirf",
"varirf_add", "varirf_cgraph", "varirf_create", "varirf_ctable",
"varirf_describe", "varirf_dir", "varirf_drop", "varirf_erase",
"varirf_graph", "varirf_ograph", "varirf_rename", "varirf_set",
"varirf_table", "varlist", "varlmar", "varnorm", "varsoc",
"varstable", "varstable_w", "varstable_w2", "varwle",
"vce", "vec", "vec_fevd", "vec_mkphi", "vec_p", "vec_p_w",
"vecirf_create", "veclmar", "veclmar_w", "vecnorm",
"vecnorm_w", "vecrank", "vecstable", "verinst", "vers",
"versi", "versio", "version", "view", "viewsource", "vif",
"vwls", "wdatetof", "webdescribe", "webseek", "webuse",
"weib1_lf", "weib2_lf", "weib_lf", "weib_lf0", "weibhet_glf",
"weibhet_glf_sh", "weibhet_glfa", "weibhet_glfa_sh",
"weibhet_gp", "weibhet_ilf", "weibhet_ilf_sh", "weibhet_ilfa",
"weibhet_ilfa_sh", "weibhet_ip", "weibu_sw", "weibul_p",
"weibull", "weibull_c", "weibull_s", "weibullhet",
"wh", "whelp", "whi", "which", "whil", "while", "wilc_st",
"wilcoxon", "win", "wind", "windo", "window", "winexec",
"wntestb", "wntestb_7", "wntestq", "xchart", "xchart_7",
"xcorr", "xcorr_7", "xi", "xi_6", "xmlsav", "xmlsave",
"xmluse", "xpose", "xsh", "xshe", "xshel", "xshell",
"xt_iis", "xt_tis", "xtab_p", "xtabond", "xtbin_p",
"xtclog", "xtcloglog", "xtcloglog_8", "xtcloglog_d2",
"xtcloglog_pa_p", "xtcloglog_re_p", "xtcnt_p", "xtcorr",
"xtdata", "xtdes", "xtfront_p", "xtfrontier", "xtgee",
"xtgee_elink", "xtgee_estat", "xtgee_makeivar", "xtgee_p",
"xtgee_plink", "xtgls", "xtgls_p", "xthaus", "xthausman",
"xtht_p", "xthtaylor", "xtile", "xtint_p", "xtintreg",
"xtintreg_8", "xtintreg_d2", "xtintreg_p", "xtivp_1",
"xtivp_2", "xtivreg", "xtline", "xtline_ex", "xtlogit",
"xtlogit_8", "xtlogit_d2", "xtlogit_fe_p", "xtlogit_pa_p",
"xtlogit_re_p", "xtmixed", "xtmixed_estat", "xtmixed_p",
"xtnb_fe", "xtnb_lf", "xtnbreg", "xtnbreg_pa_p",
"xtnbreg_refe_p", "xtpcse", "xtpcse_p", "xtpois", "xtpoisson",
"xtpoisson_d2", "xtpoisson_pa_p", "xtpoisson_refe_p", "xtpred",
"xtprobit", "xtprobit_8", "xtprobit_d2", "xtprobit_re_p",
"xtps_fe", "xtps_lf", "xtps_ren", "xtps_ren_8", "xtrar_p",
"xtrc", "xtrc_p", "xtrchh", "xtrefe_p", "xtreg", "xtreg_be",
"xtreg_fe", "xtreg_ml", "xtreg_pa_p", "xtreg_re",
"xtregar", "xtrere_p", "xtset", "xtsf_ll", "xtsf_llti",
"xtsum", "xttab", "xttest0", "xttobit", "xttobit_8",
"xttobit_p", "xttrans", "yx", "yxview__barlike_draw",
"yxview_area_draw", "yxview_bar_draw", "yxview_dot_draw",
"yxview_dropline_draw", "yxview_function_draw",
"yxview_iarrow_draw", "yxview_ilabels_draw",
"yxview_normal_draw", "yxview_pcarrow_draw",
"yxview_pcbarrow_draw", "yxview_pccapsym_draw",
"yxview_pcscatter_draw", "yxview_pcspike_draw",
"yxview_rarea_draw", "yxview_rbar_draw", "yxview_rbarm_draw",
"yxview_rcap_draw", "yxview_rcapsym_draw",
"yxview_rconnected_draw", "yxview_rline_draw",
"yxview_rscatter_draw", "yxview_rspike_draw",
"yxview_spike_draw", "yxview_sunflower_draw", "zap_s", "zinb",
"zinb_llf", "zinb_plf", "zip", "zip_llf", "zip_p", "zip_plf",
"zt_ct_5", "zt_hc_5", "zt_hcd_5", "zt_is_5", "zt_iss_5",
"zt_sho_5", "zt_smp_5", "ztbase_5", "ztcox_5", "ztdes_5",
"ztereg_5", "ztfill_5", "ztgen_5", "ztir_5", "ztjoin_5", "ztnb",
"ztnb_p", "ztp", "ztp_p", "zts_5", "ztset_5", "ztspli_5",
"ztsum_5", "zttoct_5", "ztvary_5", "ztweib_5"
)
builtins_functions = (
"Cdhms", "Chms", "Clock", "Cmdyhms", "Cofc", "Cofd", "F",
"Fden", "Ftail", "I", "J", "_caller", "abbrev", "abs", "acos",
"acosh", "asin", "asinh", "atan", "atan2", "atanh",
"autocode", "betaden", "binomial", "binomialp", "binomialtail",
"binormal", "bofd", "byteorder", "c", "ceil", "char",
"chi2", "chi2den", "chi2tail", "cholesky", "chop", "clip",
"clock", "cloglog", "cofC", "cofd", "colnumb", "colsof", "comb",
"cond", "corr", "cos", "cosh", "d", "daily", "date", "day",
"det", "dgammapda", "dgammapdada", "dgammapdadx", "dgammapdx",
"dgammapdxdx", "dhms", "diag", "diag0cnt", "digamma",
"dofC", "dofb", "dofc", "dofh", "dofm", "dofq", "dofw",
"dofy", "dow", "doy", "dunnettprob", "e", "el", "epsdouble",
"epsfloat", "exp", "fileexists", "fileread", "filereaderror",
"filewrite", "float", "floor", "fmtwidth", "gammaden",
"gammap", "gammaptail", "get", "group", "h", "hadamard",
"halfyear", "halfyearly", "has_eprop", "hh", "hhC", "hms",
"hofd", "hours", "hypergeometric", "hypergeometricp", "ibeta",
"ibetatail", "index", "indexnot", "inlist", "inrange", "int",
"inv", "invF", "invFtail", "invbinomial", "invbinomialtail",
"invchi2", "invchi2tail", "invcloglog", "invdunnettprob",
"invgammap", "invgammaptail", "invibeta", "invibetatail",
"invlogit", "invnFtail", "invnbinomial", "invnbinomialtail",
"invnchi2", "invnchi2tail", "invnibeta", "invnorm", "invnormal",
"invnttail", "invpoisson", "invpoissontail", "invsym", "invt",
"invttail", "invtukeyprob", "irecode", "issym", "issymmetric",
"itrim", "length", "ln", "lnfact", "lnfactorial", "lngamma",
"lnnormal", "lnnormalden", "log", "log10", "logit", "lower",
"ltrim", "m", "match", "matmissing", "matrix", "matuniform",
"max", "maxbyte", "maxdouble", "maxfloat", "maxint", "maxlong",
"mdy", "mdyhms", "mi", "min", "minbyte", "mindouble",
"minfloat", "minint", "minlong", "minutes", "missing", "mm",
"mmC", "mod", "mofd", "month", "monthly", "mreldif",
"msofhours", "msofminutes", "msofseconds", "nF", "nFden",
"nFtail", "nbetaden", "nbinomial", "nbinomialp", "nbinomialtail",
"nchi2", "nchi2den", "nchi2tail", "nibeta", "norm", "normal",
"normalden", "normd", "npnF", "npnchi2", "npnt", "nt", "ntden",
"nttail", "nullmat", "plural", "poisson", "poissonp",
"poissontail", "proper", "q", "qofd", "quarter", "quarterly",
"r", "rbeta", "rbinomial", "rchi2", "real", "recode", "regexm",
"regexr", "regexs", "reldif", "replay", "return", "reverse",
"rgamma", "rhypergeometric", "rnbinomial", "rnormal", "round",
"rownumb", "rowsof", "rpoisson", "rt", "rtrim", "runiform", "s",
"scalar", "seconds", "sign", "sin", "sinh", "smallestdouble",
"soundex", "soundex_nara", "sqrt", "ss", "ssC", "strcat",
"strdup", "string", "strlen", "strlower", "strltrim", "strmatch",
"strofreal", "strpos", "strproper", "strreverse", "strrtrim",
"strtoname", "strtrim", "strupper", "subinstr", "subinword",
"substr", "sum", "sweep", "syminv", "t", "tC", "tan", "tanh",
"tc", "td", "tden", "th", "tin", "tm", "tq", "trace",
"trigamma", "trim", "trunc", "ttail", "tukeyprob", "tw",
"twithin", "uniform", "upper", "vec", "vecdiag", "w", "week",
"weekly", "wofd", "word", "wordcount", "year", "yearly",
"yh", "ym", "yofd", "yq", "yw"
)

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1,240 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexers.actionscript
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Lexers for ActionScript and MXML.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import re
from pygments.lexer import RegexLexer, bygroups, using, this, words, default
from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
Number, Punctuation
__all__ = ['ActionScriptLexer', 'ActionScript3Lexer', 'MxmlLexer']
class ActionScriptLexer(RegexLexer):
"""
For ActionScript source code.
.. versionadded:: 0.9
"""
name = 'ActionScript'
aliases = ['as', 'actionscript']
filenames = ['*.as']
mimetypes = ['application/x-actionscript', 'text/x-actionscript',
'text/actionscript']
flags = re.DOTALL
tokens = {
'root': [
(r'\s+', Text),
(r'//.*?\n', Comment.Single),
(r'/\*.*?\*/', Comment.Multiline),
(r'/(\\\\|\\/|[^/\n])*/[gim]*', String.Regex),
(r'[~^*!%&<>|+=:;,/?\\-]+', Operator),
(r'[{}\[\]();.]+', Punctuation),
(words((
'case', 'default', 'for', 'each', 'in', 'while', 'do', 'break',
'return', 'continue', 'if', 'else', 'throw', 'try', 'catch',
'var', 'with', 'new', 'typeof', 'arguments', 'instanceof', 'this',
'switch'), suffix=r'\b'),
Keyword),
(words((
'class', 'public', 'final', 'internal', 'native', 'override', 'private',
'protected', 'static', 'import', 'extends', 'implements', 'interface',
'intrinsic', 'return', 'super', 'dynamic', 'function', 'const', 'get',
'namespace', 'package', 'set'), suffix=r'\b'),
Keyword.Declaration),
(r'(true|false|null|NaN|Infinity|-Infinity|undefined|Void)\b',
Keyword.Constant),
(words((
'Accessibility', 'AccessibilityProperties', 'ActionScriptVersion',
'ActivityEvent', 'AntiAliasType', 'ApplicationDomain', 'AsBroadcaster', 'Array',
'AsyncErrorEvent', 'AVM1Movie', 'BevelFilter', 'Bitmap', 'BitmapData',
'BitmapDataChannel', 'BitmapFilter', 'BitmapFilterQuality', 'BitmapFilterType',
'BlendMode', 'BlurFilter', 'Boolean', 'ByteArray', 'Camera', 'Capabilities', 'CapsStyle',
'Class', 'Color', 'ColorMatrixFilter', 'ColorTransform', 'ContextMenu',
'ContextMenuBuiltInItems', 'ContextMenuEvent', 'ContextMenuItem',
'ConvultionFilter', 'CSMSettings', 'DataEvent', 'Date', 'DefinitionError',
'DeleteObjectSample', 'Dictionary', 'DisplacmentMapFilter', 'DisplayObject',
'DisplacmentMapFilterMode', 'DisplayObjectContainer', 'DropShadowFilter',
'Endian', 'EOFError', 'Error', 'ErrorEvent', 'EvalError', 'Event', 'EventDispatcher',
'EventPhase', 'ExternalInterface', 'FileFilter', 'FileReference',
'FileReferenceList', 'FocusDirection', 'FocusEvent', 'Font', 'FontStyle', 'FontType',
'FrameLabel', 'FullScreenEvent', 'Function', 'GlowFilter', 'GradientBevelFilter',
'GradientGlowFilter', 'GradientType', 'Graphics', 'GridFitType', 'HTTPStatusEvent',
'IBitmapDrawable', 'ID3Info', 'IDataInput', 'IDataOutput', 'IDynamicPropertyOutput'
'IDynamicPropertyWriter', 'IEventDispatcher', 'IExternalizable',
'IllegalOperationError', 'IME', 'IMEConversionMode', 'IMEEvent', 'int',
'InteractiveObject', 'InterpolationMethod', 'InvalidSWFError', 'InvokeEvent',
'IOError', 'IOErrorEvent', 'JointStyle', 'Key', 'Keyboard', 'KeyboardEvent', 'KeyLocation',
'LineScaleMode', 'Loader', 'LoaderContext', 'LoaderInfo', 'LoadVars', 'LocalConnection',
'Locale', 'Math', 'Matrix', 'MemoryError', 'Microphone', 'MorphShape', 'Mouse', 'MouseEvent',
'MovieClip', 'MovieClipLoader', 'Namespace', 'NetConnection', 'NetStatusEvent',
'NetStream', 'NewObjectSample', 'Number', 'Object', 'ObjectEncoding', 'PixelSnapping',
'Point', 'PrintJob', 'PrintJobOptions', 'PrintJobOrientation', 'ProgressEvent', 'Proxy',
'QName', 'RangeError', 'Rectangle', 'ReferenceError', 'RegExp', 'Responder', 'Sample',
'Scene', 'ScriptTimeoutError', 'Security', 'SecurityDomain', 'SecurityError',
'SecurityErrorEvent', 'SecurityPanel', 'Selection', 'Shape', 'SharedObject',
'SharedObjectFlushStatus', 'SimpleButton', 'Socket', 'Sound', 'SoundChannel',
'SoundLoaderContext', 'SoundMixer', 'SoundTransform', 'SpreadMethod', 'Sprite',
'StackFrame', 'StackOverflowError', 'Stage', 'StageAlign', 'StageDisplayState',
'StageQuality', 'StageScaleMode', 'StaticText', 'StatusEvent', 'String', 'StyleSheet',
'SWFVersion', 'SyncEvent', 'SyntaxError', 'System', 'TextColorType', 'TextField',
'TextFieldAutoSize', 'TextFieldType', 'TextFormat', 'TextFormatAlign',
'TextLineMetrics', 'TextRenderer', 'TextSnapshot', 'Timer', 'TimerEvent', 'Transform',
'TypeError', 'uint', 'URIError', 'URLLoader', 'URLLoaderDataFormat', 'URLRequest',
'URLRequestHeader', 'URLRequestMethod', 'URLStream', 'URLVariabeles', 'VerifyError',
'Video', 'XML', 'XMLDocument', 'XMLList', 'XMLNode', 'XMLNodeType', 'XMLSocket',
'XMLUI'), suffix=r'\b'),
Name.Builtin),
(words((
'decodeURI', 'decodeURIComponent', 'encodeURI', 'escape', 'eval', 'isFinite', 'isNaN',
'isXMLName', 'clearInterval', 'fscommand', 'getTimer', 'getURL', 'getVersion',
'parseFloat', 'parseInt', 'setInterval', 'trace', 'updateAfterEvent',
'unescape'), suffix=r'\b'),
Name.Function),
(r'[$a-zA-Z_]\w*', Name.Other),
(r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
(r'0x[0-9a-f]+', Number.Hex),
(r'[0-9]+', Number.Integer),
(r'"(\\\\|\\"|[^"])*"', String.Double),
(r"'(\\\\|\\'|[^'])*'", String.Single),
]
}
class ActionScript3Lexer(RegexLexer):
"""
For ActionScript 3 source code.
.. versionadded:: 0.11
"""
name = 'ActionScript 3'
aliases = ['as3', 'actionscript3']
filenames = ['*.as']
mimetypes = ['application/x-actionscript3', 'text/x-actionscript3',
'text/actionscript3']
identifier = r'[$a-zA-Z_]\w*'
typeidentifier = identifier + '(?:\.<\w+>)?'
flags = re.DOTALL | re.MULTILINE
tokens = {
'root': [
(r'\s+', Text),
(r'(function\s+)(' + identifier + r')(\s*)(\()',
bygroups(Keyword.Declaration, Name.Function, Text, Operator),
'funcparams'),
(r'(var|const)(\s+)(' + identifier + r')(\s*)(:)(\s*)(' +
typeidentifier + r')',
bygroups(Keyword.Declaration, Text, Name, Text, Punctuation, Text,
Keyword.Type)),
(r'(import|package)(\s+)((?:' + identifier + r'|\.)+)(\s*)',
bygroups(Keyword, Text, Name.Namespace, Text)),
(r'(new)(\s+)(' + typeidentifier + r')(\s*)(\()',
bygroups(Keyword, Text, Keyword.Type, Text, Operator)),
(r'//.*?\n', Comment.Single),
(r'/\*.*?\*/', Comment.Multiline),
(r'/(\\\\|\\/|[^\n])*/[gisx]*', String.Regex),
(r'(\.)(' + identifier + r')', bygroups(Operator, Name.Attribute)),
(r'(case|default|for|each|in|while|do|break|return|continue|if|else|'
r'throw|try|catch|with|new|typeof|arguments|instanceof|this|'
r'switch|import|include|as|is)\b',
Keyword),
(r'(class|public|final|internal|native|override|private|protected|'
r'static|import|extends|implements|interface|intrinsic|return|super|'
r'dynamic|function|const|get|namespace|package|set)\b',
Keyword.Declaration),
(r'(true|false|null|NaN|Infinity|-Infinity|undefined|void)\b',
Keyword.Constant),
(r'(decodeURI|decodeURIComponent|encodeURI|escape|eval|isFinite|isNaN|'
r'isXMLName|clearInterval|fscommand|getTimer|getURL|getVersion|'
r'isFinite|parseFloat|parseInt|setInterval|trace|updateAfterEvent|'
r'unescape)\b', Name.Function),
(identifier, Name),
(r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
(r'0x[0-9a-f]+', Number.Hex),
(r'[0-9]+', Number.Integer),
(r'"(\\\\|\\"|[^"])*"', String.Double),
(r"'(\\\\|\\'|[^'])*'", String.Single),
(r'[~^*!%&<>|+=:;,/?\\{}\[\]().-]+', Operator),
],
'funcparams': [
(r'\s+', Text),
(r'(\s*)(\.\.\.)?(' + identifier + r')(\s*)(:)(\s*)(' +
typeidentifier + r'|\*)(\s*)',
bygroups(Text, Punctuation, Name, Text, Operator, Text,
Keyword.Type, Text), 'defval'),
(r'\)', Operator, 'type')
],
'type': [
(r'(\s*)(:)(\s*)(' + typeidentifier + r'|\*)',
bygroups(Text, Operator, Text, Keyword.Type), '#pop:2'),
(r'\s+', Text, '#pop:2'),
default('#pop:2')
],
'defval': [
(r'(=)(\s*)([^(),]+)(\s*)(,?)',
bygroups(Operator, Text, using(this), Text, Operator), '#pop'),
(r',', Operator, '#pop'),
default('#pop')
]
}
def analyse_text(text):
if re.match(r'\w+\s*:\s*\w', text):
return 0.3
return 0
class MxmlLexer(RegexLexer):
"""
For MXML markup.
Nested AS3 in <script> tags is highlighted by the appropriate lexer.
.. versionadded:: 1.1
"""
flags = re.MULTILINE | re.DOTALL
name = 'MXML'
aliases = ['mxml']
filenames = ['*.mxml']
mimetimes = ['text/xml', 'application/xml']
tokens = {
'root': [
('[^<&]+', Text),
(r'&\S*?;', Name.Entity),
(r'(\<\!\[CDATA\[)(.*?)(\]\]\>)',
bygroups(String, using(ActionScript3Lexer), String)),
('<!--', Comment, 'comment'),
(r'<\?.*?\?>', Comment.Preproc),
('<![^>]*>', Comment.Preproc),
(r'<\s*[\w:.-]+', Name.Tag, 'tag'),
(r'<\s*/\s*[\w:.-]+\s*>', Name.Tag),
],
'comment': [
('[^-]+', Comment),
('-->', Comment, '#pop'),
('-', Comment),
],
'tag': [
(r'\s+', Text),
(r'[\w.:-]+\s*=', Name.Attribute, 'attr'),
(r'/?\s*>', Name.Tag, '#pop'),
],
'attr': [
('\s+', Text),
('".*?"', String, '#pop'),
("'.*?'", String, '#pop'),
(r'[^\s>]+', String, '#pop'),
],
}

View File

@ -1,24 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexers.agile
~~~~~~~~~~~~~~~~~~~~~
Just export lexer classes previously contained in this module.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from pygments.lexers.lisp import SchemeLexer
from pygments.lexers.jvm import IokeLexer, ClojureLexer
from pygments.lexers.python import PythonLexer, PythonConsoleLexer, \
PythonTracebackLexer, Python3Lexer, Python3TracebackLexer, DgLexer
from pygments.lexers.ruby import RubyLexer, RubyConsoleLexer, FancyLexer
from pygments.lexers.perl import PerlLexer, Perl6Lexer
from pygments.lexers.d import CrocLexer, MiniDLexer
from pygments.lexers.iolang import IoLexer
from pygments.lexers.tcl import TclLexer
from pygments.lexers.factor import FactorLexer
from pygments.lexers.scripting import LuaLexer, MoonScriptLexer
__all__ = []

View File

@ -1,221 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexers.algebra
~~~~~~~~~~~~~~~~~~~~~~~
Lexers for computer algebra systems.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import re
from pygments.lexer import RegexLexer, bygroups, words
from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
Number, Punctuation
__all__ = ['GAPLexer', 'MathematicaLexer', 'MuPADLexer', 'BCLexer']
class GAPLexer(RegexLexer):
"""
For `GAP <http://www.gap-system.org>`_ source code.
.. versionadded:: 2.0
"""
name = 'GAP'
aliases = ['gap']
filenames = ['*.g', '*.gd', '*.gi', '*.gap']
tokens = {
'root': [
(r'#.*$', Comment.Single),
(r'"(?:[^"\\]|\\.)*"', String),
(r'\(|\)|\[|\]|\{|\}', Punctuation),
(r'''(?x)\b(?:
if|then|elif|else|fi|
for|while|do|od|
repeat|until|
break|continue|
function|local|return|end|
rec|
quit|QUIT|
IsBound|Unbind|
TryNextMethod|
Info|Assert
)\b''', Keyword),
(r'''(?x)\b(?:
true|false|fail|infinity
)\b''',
Name.Constant),
(r'''(?x)\b(?:
(Declare|Install)([A-Z][A-Za-z]+)|
BindGlobal|BIND_GLOBAL
)\b''',
Name.Builtin),
(r'\.|,|:=|;|=|\+|-|\*|/|\^|>|<', Operator),
(r'''(?x)\b(?:
and|or|not|mod|in
)\b''',
Operator.Word),
(r'''(?x)
(?:\w+|`[^`]*`)
(?:::\w+|`[^`]*`)*''', Name.Variable),
(r'[0-9]+(?:\.[0-9]*)?(?:e[0-9]+)?', Number),
(r'\.[0-9]+(?:e[0-9]+)?', Number),
(r'.', Text)
],
}
class MathematicaLexer(RegexLexer):
"""
Lexer for `Mathematica <http://www.wolfram.com/mathematica/>`_ source code.
.. versionadded:: 2.0
"""
name = 'Mathematica'
aliases = ['mathematica', 'mma', 'nb']
filenames = ['*.nb', '*.cdf', '*.nbp', '*.ma']
mimetypes = ['application/mathematica',
'application/vnd.wolfram.mathematica',
'application/vnd.wolfram.mathematica.package',
'application/vnd.wolfram.cdf']
# http://reference.wolfram.com/mathematica/guide/Syntax.html
operators = (
";;", "=", "=.", "!=" "==", ":=", "->", ":>", "/.", "+", "-", "*", "/",
"^", "&&", "||", "!", "<>", "|", "/;", "?", "@", "//", "/@", "@@",
"@@@", "~~", "===", "&", "<", ">", "<=", ">=",
)
punctuation = (",", ";", "(", ")", "[", "]", "{", "}")
def _multi_escape(entries):
return '(%s)' % ('|'.join(re.escape(entry) for entry in entries))
tokens = {
'root': [
(r'(?s)\(\*.*?\*\)', Comment),
(r'([a-zA-Z]+[A-Za-z0-9]*`)', Name.Namespace),
(r'([A-Za-z0-9]*_+[A-Za-z0-9]*)', Name.Variable),
(r'#\d*', Name.Variable),
(r'([a-zA-Z]+[a-zA-Z0-9]*)', Name),
(r'-?\d+\.\d*', Number.Float),
(r'-?\d*\.\d+', Number.Float),
(r'-?\d+', Number.Integer),
(words(operators), Operator),
(words(punctuation), Punctuation),
(r'".*?"', String),
(r'\s+', Text.Whitespace),
],
}
class MuPADLexer(RegexLexer):
"""
A `MuPAD <http://www.mupad.com>`_ lexer.
Contributed by Christopher Creutzig <christopher@creutzig.de>.
.. versionadded:: 0.8
"""
name = 'MuPAD'
aliases = ['mupad']
filenames = ['*.mu']
tokens = {
'root': [
(r'//.*?$', Comment.Single),
(r'/\*', Comment.Multiline, 'comment'),
(r'"(?:[^"\\]|\\.)*"', String),
(r'\(|\)|\[|\]|\{|\}', Punctuation),
(r'''(?x)\b(?:
next|break|end|
axiom|end_axiom|category|end_category|domain|end_domain|inherits|
if|%if|then|elif|else|end_if|
case|of|do|otherwise|end_case|
while|end_while|
repeat|until|end_repeat|
for|from|to|downto|step|end_for|
proc|local|option|save|begin|end_proc|
delete|frame
)\b''', Keyword),
(r'''(?x)\b(?:
DOM_ARRAY|DOM_BOOL|DOM_COMPLEX|DOM_DOMAIN|DOM_EXEC|DOM_EXPR|
DOM_FAIL|DOM_FLOAT|DOM_FRAME|DOM_FUNC_ENV|DOM_HFARRAY|DOM_IDENT|
DOM_INT|DOM_INTERVAL|DOM_LIST|DOM_NIL|DOM_NULL|DOM_POLY|DOM_PROC|
DOM_PROC_ENV|DOM_RAT|DOM_SET|DOM_STRING|DOM_TABLE|DOM_VAR
)\b''', Name.Class),
(r'''(?x)\b(?:
PI|EULER|E|CATALAN|
NIL|FAIL|undefined|infinity|
TRUE|FALSE|UNKNOWN
)\b''',
Name.Constant),
(r'\b(?:dom|procname)\b', Name.Builtin.Pseudo),
(r'\.|,|:|;|=|\+|-|\*|/|\^|@|>|<|\$|\||!|\'|%|~=', Operator),
(r'''(?x)\b(?:
and|or|not|xor|
assuming|
div|mod|
union|minus|intersect|in|subset
)\b''',
Operator.Word),
(r'\b(?:I|RDN_INF|RD_NINF|RD_NAN)\b', Number),
# (r'\b(?:adt|linalg|newDomain|hold)\b', Name.Builtin),
(r'''(?x)
((?:[a-zA-Z_#][\w#]*|`[^`]*`)
(?:::[a-zA-Z_#][\w#]*|`[^`]*`)*)(\s*)([(])''',
bygroups(Name.Function, Text, Punctuation)),
(r'''(?x)
(?:[a-zA-Z_#][\w#]*|`[^`]*`)
(?:::[a-zA-Z_#][\w#]*|`[^`]*`)*''', Name.Variable),
(r'[0-9]+(?:\.[0-9]*)?(?:e[0-9]+)?', Number),
(r'\.[0-9]+(?:e[0-9]+)?', Number),
(r'.', Text)
],
'comment': [
(r'[^*/]', Comment.Multiline),
(r'/\*', Comment.Multiline, '#push'),
(r'\*/', Comment.Multiline, '#pop'),
(r'[*/]', Comment.Multiline)
],
}
class BCLexer(RegexLexer):
"""
A `BC <https://www.gnu.org/software/bc/>`_ lexer.
.. versionadded:: 2.1
"""
name = 'BC'
aliases = ['bc']
filenames = ['*.bc']
tokens = {
'root': [
(r'/\*', Comment.Multiline, 'comment'),
(r'"(?:[^"\\]|\\.)*"', String),
(r'[{}();,]', Punctuation),
(words(('if', 'else', 'while', 'for', 'break', 'continue',
'halt', 'return', 'define', 'auto', 'print', 'read',
'length', 'scale', 'sqrt', 'limits', 'quit',
'warranty'), suffix=r'\b'), Keyword),
(r'\+\+|--|\|\||&&|'
r'([-<>+*%\^/!=])=?', Operator),
# bc doesn't support exponential
(r'[0-9]+(\.[0-9]*)?', Number),
(r'\.[0-9]+', Number),
(r'.', Text)
],
'comment': [
(r'[^*/]+', Comment.Multiline),
(r'\*/', Comment.Multiline, '#pop'),
(r'[*/]', Comment.Multiline)
],
}

View File

@ -1,76 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexers.ambient
~~~~~~~~~~~~~~~~~~~~~~~
Lexers for AmbientTalk language.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import re
from pygments.lexer import RegexLexer, include, words
from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
Number, Punctuation
__all__ = ['AmbientTalkLexer']
class AmbientTalkLexer(RegexLexer):
"""
Lexer for `AmbientTalk <https://code.google.com/p/ambienttalk>`_ source code.
.. versionadded:: 2.0
"""
name = 'AmbientTalk'
filenames = ['*.at']
aliases = ['at', 'ambienttalk', 'ambienttalk/2']
mimetypes = ['text/x-ambienttalk']
flags = re.MULTILINE | re.DOTALL
builtin = words(('if:', 'then:', 'else:', 'when:', 'whenever:', 'discovered:',
'disconnected:', 'reconnected:', 'takenOffline:', 'becomes:',
'export:', 'as:', 'object:', 'actor:', 'mirror:', 'taggedAs:',
'mirroredBy:', 'is:'))
tokens = {
'root': [
(r'\s+', Text),
(r'//.*?\n', Comment.Single),
(r'/\*.*?\*/', Comment.Multiline),
(r'(def|deftype|import|alias|exclude)\b', Keyword),
(builtin, Name.Builtin),
(r'(true|false|nil)\b', Keyword.Constant),
(r'(~|lobby|jlobby|/)\.', Keyword.Constant, 'namespace'),
(r'"(\\\\|\\"|[^"])*"', String),
(r'\|', Punctuation, 'arglist'),
(r'<:|[*^!%&<>+=,./?-]|:=', Operator),
(r"`[a-zA-Z_]\w*", String.Symbol),
(r"[a-zA-Z_]\w*:", Name.Function),
(r"[{}()\[\];`]", Punctuation),
(r'(self|super)\b', Name.Variable.Instance),
(r"[a-zA-Z_]\w*", Name.Variable),
(r"@[a-zA-Z_]\w*", Name.Class),
(r"@\[", Name.Class, 'annotations'),
include('numbers'),
],
'numbers': [
(r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
(r'\d+', Number.Integer)
],
'namespace': [
(r'[a-zA-Z_]\w*\.', Name.Namespace),
(r'[a-zA-Z_]\w*:', Name.Function, '#pop'),
(r'[a-zA-Z_]\w*(?!\.)', Name.Function, '#pop')
],
'annotations': [
(r"(.*?)\]", Name.Class, '#pop')
],
'arglist': [
(r'\|', Punctuation, '#pop'),
(r'\s*(,)\s*', Punctuation),
(r'[a-zA-Z_]\w*', Name.Variable),
],
}

View File

@ -1,87 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexers.ampl
~~~~~~~~~~~~~~~~~~~~
Lexers for the ampl language. <http://ampl.com/>
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from pygments.lexer import RegexLexer, bygroups, using, this, words
from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
Number, Punctuation
__all__ = ['AmplLexer']
class AmplLexer(RegexLexer):
"""
For AMPL source code.
.. versionadded:: 2.2
"""
name = 'Ampl'
aliases = ['ampl']
filenames = ['*.run']
tokens = {
'root': [
(r'\n', Text),
(r'\s+', Text.Whitespace),
(r'#.*?\n', Comment.Single),
(r'/[*](.|\n)*?[*]/', Comment.Multiline),
(words((
'call', 'cd', 'close', 'commands', 'data', 'delete', 'display',
'drop', 'end', 'environ', 'exit', 'expand', 'include', 'load',
'model', 'objective', 'option', 'problem', 'purge', 'quit',
'redeclare', 'reload', 'remove', 'reset', 'restore', 'shell',
'show', 'solexpand', 'solution', 'solve', 'update', 'unload',
'xref', 'coeff', 'coef', 'cover', 'obj', 'interval', 'default',
'from', 'to', 'to_come', 'net_in', 'net_out', 'dimen',
'dimension', 'check', 'complements', 'write', 'function',
'pipe', 'format', 'if', 'then', 'else', 'in', 'while', 'repeat',
'for'), suffix=r'\b'), Keyword.Reserved),
(r'(integer|binary|symbolic|ordered|circular|reversed|INOUT|IN|OUT|LOCAL)',
Keyword.Type),
(r'\".*?\"', String.Double),
(r'\'.*?\'', String.Single),
(r'[()\[\]{},;:]+', Punctuation),
(r'\b(\w+)(\.)(astatus|init0|init|lb0|lb1|lb2|lb|lrc|'
r'lslack|rc|relax|slack|sstatus|status|ub0|ub1|ub2|'
r'ub|urc|uslack|val)',
bygroups(Name.Variable, Punctuation, Keyword.Reserved)),
(r'(set|param|var|arc|minimize|maximize|subject to|s\.t\.|subj to|'
r'node|table|suffix|read table|write table)(\s+)(\w+)',
bygroups(Keyword.Declaration, Text, Name.Variable)),
(r'(param)(\s*)(:)(\s*)(\w+)(\s*)(:)(\s*)((\w|\s)+)',
bygroups(Keyword.Declaration, Text, Punctuation, Text,
Name.Variable, Text, Punctuation, Text, Name.Variable)),
(r'(let|fix|unfix)(\s*)((?:\{.*\})?)(\s*)(\w+)',
bygroups(Keyword.Declaration, Text, using(this), Text, Name.Variable)),
(words((
'abs', 'acos', 'acosh', 'alias', 'asin', 'asinh', 'atan', 'atan2',
'atanh', 'ceil', 'ctime', 'cos', 'exp', 'floor', 'log', 'log10',
'max', 'min', 'precision', 'round', 'sin', 'sinh', 'sqrt', 'tan',
'tanh', 'time', 'trunc', 'Beta', 'Cauchy', 'Exponential', 'Gamma',
'Irand224', 'Normal', 'Normal01', 'Poisson', 'Uniform', 'Uniform01',
'num', 'num0', 'ichar', 'char', 'length', 'substr', 'sprintf',
'match', 'sub', 'gsub', 'print', 'printf', 'next', 'nextw', 'prev',
'prevw', 'first', 'last', 'ord', 'ord0', 'card', 'arity',
'indexarity'), prefix=r'\b', suffix=r'\b'), Name.Builtin),
(r'(\+|\-|\*|/|\*\*|=|<=|>=|==|\||\^|<|>|\!|\.\.|:=|\&|\!=|<<|>>)',
Operator),
(words((
'or', 'exists', 'forall', 'and', 'in', 'not', 'within', 'union',
'diff', 'difference', 'symdiff', 'inter', 'intersect',
'intersection', 'cross', 'setof', 'by', 'less', 'sum', 'prod',
'product', 'div', 'mod'), suffix=r'\b'),
Keyword.Reserved), # Operator.Name but not enough emphasized with that
(r'(\d+\.(?!\.)\d*|\.(?!.)\d+)([eE][+-]?\d+)?', Number.Float),
(r'\d+([eE][+-]?\d+)?', Number.Integer),
(r'[+-]?Infinity', Number.Integer),
(r'(\w+|(\.(?!\.)))', Text)
]
}

View File

@ -1,101 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexers.apl
~~~~~~~~~~~~~~~~~~~
Lexers for APL.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from pygments.lexer import RegexLexer
from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
Number, Punctuation
__all__ = ['APLLexer']
class APLLexer(RegexLexer):
"""
A simple APL lexer.
.. versionadded:: 2.0
"""
name = 'APL'
aliases = ['apl']
filenames = ['*.apl']
tokens = {
'root': [
# Whitespace
# ==========
(r'\s+', Text),
#
# Comment
# =======
# '⍝' is traditional; '#' is supported by GNU APL and NGN (but not Dyalog)
(u'[⍝#].*$', Comment.Single),
#
# Strings
# =======
(r'\'((\'\')|[^\'])*\'', String.Single),
(r'"(("")|[^"])*"', String.Double), # supported by NGN APL
#
# Punctuation
# ===========
# This token type is used for diamond and parenthesis
# but not for bracket and ; (see below)
(u'[⋄◇()]', Punctuation),
#
# Array indexing
# ==============
# Since this token type is very important in APL, it is not included in
# the punctuation token type but rather in the following one
(r'[\[\];]', String.Regex),
#
# Distinguished names
# ===================
# following IBM APL2 standard
(u'⎕[A-Za-zΔ∆⍙][A-Za-zΔ∆⍙_¯0-9]*', Name.Function),
#
# Labels
# ======
# following IBM APL2 standard
# (u'[A-Za-zΔ∆⍙][A-Za-zΔ∆⍙_¯0-9]*:', Name.Label),
#
# Variables
# =========
# following IBM APL2 standard
(u'[A-Za-zΔ∆⍙][A-Za-zΔ∆⍙_¯0-9]*', Name.Variable),
#
# Numbers
# =======
(u'¯?(0[Xx][0-9A-Fa-f]+|[0-9]*\.?[0-9]+([Ee][+¯]?[0-9]+)?|¯|∞)'
u'([Jj]¯?(0[Xx][0-9A-Fa-f]+|[0-9]*\.?[0-9]+([Ee][+¯]?[0-9]+)?|¯|∞))?',
Number),
#
# Operators
# ==========
(u'[\.\\\/⌿⍀¨⍣⍨⍠⍤∘]', Name.Attribute), # closest token type
(u'[+\-×÷⌈⌊∣|?*⍟○!⌹<≤=>≥≠≡≢∊⍷∪∩~∨∧⍱⍲⍴,⍪⌽⊖⍉↑↓⊂⊃⌷⍋⍒⊤⊥⍕⍎⊣⊢⍁⍂≈⌸⍯↗]',
Operator),
#
# Constant
# ========
(u'', Name.Constant),
#
# Quad symbol
# ===========
(u'[⎕⍞]', Name.Variable.Global),
#
# Arrows left/right
# =================
(u'[←→]', Keyword.Declaration),
#
# D-Fn
# ====
(u'[⍺⍵⍶⍹∇:]', Name.Builtin.Pseudo),
(r'[{}]', Keyword.Type),
],
}

View File

@ -1,318 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexers.archetype
~~~~~~~~~~~~~~~~~~~~~~~~~
Lexer for Archetype-related syntaxes, including:
- ODIN syntax <https://github.com/openEHR/odin>
- ADL syntax <http://www.openehr.org/releases/trunk/architecture/am/adl2.pdf>
- cADL sub-syntax of ADL
For uses of this syntax, see the openEHR archetypes <http://www.openEHR.org/ckm>
Contributed by Thomas Beale <https://github.com/wolandscat>,
<https://bitbucket.org/thomas_beale>.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from pygments.lexer import RegexLexer, include, bygroups, using, default
from pygments.token import Text, Comment, Name, Literal, Number, String, \
Punctuation, Keyword, Operator, Generic
__all__ = ['OdinLexer', 'CadlLexer', 'AdlLexer']
class AtomsLexer(RegexLexer):
"""
Lexer for Values used in ADL and ODIN.
.. versionadded:: 2.1
"""
tokens = {
# ----- pseudo-states for inclusion -----
'whitespace': [
(r'\n', Text),
(r'\s+', Text),
(r'[ \t]*--.*$', Comment),
],
'archetype_id': [
(r'[ \t]*([a-zA-Z]\w+(\.[a-zA-Z]\w+)*::)?[a-zA-Z]\w+(-[a-zA-Z]\w+){2}'
r'\.\w+[\w-]*\.v\d+(\.\d+){,2}((-[a-z]+)(\.\d+)?)?', Name.Decorator),
],
'date_constraints': [
# ISO 8601-based date/time constraints
(r'[Xx?YyMmDdHhSs\d]{2,4}([:-][Xx?YyMmDdHhSs\d]{2}){2}', Literal.Date),
# ISO 8601-based duration constraints + optional trailing slash
(r'(P[YyMmWwDd]+(T[HhMmSs]+)?|PT[HhMmSs]+)/?', Literal.Date),
],
'ordered_values': [
# ISO 8601 date with optional 'T' ligature
(r'\d{4}-\d{2}-\d{2}T?', Literal.Date),
# ISO 8601 time
(r'\d{2}:\d{2}:\d{2}(\.\d+)?([+-]\d{4}|Z)?', Literal.Date),
# ISO 8601 duration
(r'P((\d*(\.\d+)?[YyMmWwDd]){1,3}(T(\d*(\.\d+)?[HhMmSs]){,3})?|'
r'T(\d*(\.\d+)?[HhMmSs]){,3})', Literal.Date),
(r'[+-]?(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+', Number.Float),
(r'[+-]?(\d+)*\.\d+%?', Number.Float),
(r'0x[0-9a-fA-F]+', Number.Hex),
(r'[+-]?\d+%?', Number.Integer),
],
'values': [
include('ordered_values'),
(r'([Tt]rue|[Ff]alse)', Literal),
(r'"', String, 'string'),
(r"'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char),
(r'[a-z][a-z0-9+.-]*:', Literal, 'uri'),
# term code
(r'(\[)(\w[\w-]*(?:\([^)\n]+\))?)(::)(\w[\w-]*)(\])',
bygroups(Punctuation, Name.Decorator, Punctuation, Name.Decorator,
Punctuation)),
(r'\|', Punctuation, 'interval'),
# list continuation
(r'\.\.\.', Punctuation),
],
'constraint_values': [
(r'(\[)(\w[\w-]*(?:\([^)\n]+\))?)(::)',
bygroups(Punctuation, Name.Decorator, Punctuation), 'adl14_code_constraint'),
# ADL 1.4 ordinal constraint
(r'(\d*)(\|)(\[\w[\w-]*::\w[\w-]*\])((?:[,;])?)',
bygroups(Number, Punctuation, Name.Decorator, Punctuation)),
include('date_constraints'),
include('values'),
],
# ----- real states -----
'string': [
('"', String, '#pop'),
(r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|'
r'u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|[0-7]{1,3})', String.Escape),
# all other characters
(r'[^\\"]+', String),
# stray backslash
(r'\\', String),
],
'uri': [
# effective URI terminators
(r'[,>\s]', Punctuation, '#pop'),
(r'[^>\s,]+', Literal),
],
'interval': [
(r'\|', Punctuation, '#pop'),
include('ordered_values'),
(r'\.\.', Punctuation),
(r'[<>=] *', Punctuation),
# handle +/-
(r'\+/-', Punctuation),
(r'\s+', Text),
],
'any_code': [
include('archetype_id'),
# if it is a code
(r'[a-z_]\w*[0-9.]+(@[^\]]+)?', Name.Decorator),
# if it is tuple with attribute names
(r'[a-z_]\w*', Name.Class),
# if it is an integer, i.e. Xpath child index
(r'[0-9]+', Text),
(r'\|', Punctuation, 'code_rubric'),
(r'\]', Punctuation, '#pop'),
# handle use_archetype statement
(r'\s*,\s*', Punctuation),
],
'code_rubric': [
(r'\|', Punctuation, '#pop'),
(r'[^|]+', String),
],
'adl14_code_constraint': [
(r'\]', Punctuation, '#pop'),
(r'\|', Punctuation, 'code_rubric'),
(r'(\w[\w-]*)([;,]?)', bygroups(Name.Decorator, Punctuation)),
include('whitespace'),
],
}
class OdinLexer(AtomsLexer):
"""
Lexer for ODIN syntax.
.. versionadded:: 2.1
"""
name = 'ODIN'
aliases = ['odin']
filenames = ['*.odin']
mimetypes = ['text/odin']
tokens = {
'path': [
(r'>', Punctuation, '#pop'),
# attribute name
(r'[a-z_]\w*', Name.Class),
(r'/', Punctuation),
(r'\[', Punctuation, 'key'),
(r'\s*,\s*', Punctuation, '#pop'),
(r'\s+', Text, '#pop'),
],
'key': [
include('values'),
(r'\]', Punctuation, '#pop'),
],
'type_cast': [
(r'\)', Punctuation, '#pop'),
(r'[^)]+', Name.Class),
],
'root': [
include('whitespace'),
(r'([Tt]rue|[Ff]alse)', Literal),
include('values'),
# x-ref path
(r'/', Punctuation, 'path'),
# x-ref path starting with key
(r'\[', Punctuation, 'key'),
# attribute name
(r'[a-z_]\w*', Name.Class),
(r'=', Operator),
(r'\(', Punctuation, 'type_cast'),
(r',', Punctuation),
(r'<', Punctuation),
(r'>', Punctuation),
(r';', Punctuation),
],
}
class CadlLexer(AtomsLexer):
"""
Lexer for cADL syntax.
.. versionadded:: 2.1
"""
name = 'cADL'
aliases = ['cadl']
filenames = ['*.cadl']
tokens = {
'path': [
# attribute name
(r'[a-z_]\w*', Name.Class),
(r'/', Punctuation),
(r'\[', Punctuation, 'any_code'),
(r'\s+', Punctuation, '#pop'),
],
'root': [
include('whitespace'),
(r'(cardinality|existence|occurrences|group|include|exclude|'
r'allow_archetype|use_archetype|use_node)\W', Keyword.Type),
(r'(and|or|not|there_exists|xor|implies|for_all)\W', Keyword.Type),
(r'(after|before|closed)\W', Keyword.Type),
(r'(not)\W', Operator),
(r'(matches|is_in)\W', Operator),
# is_in / not is_in char
(u'(\u2208|\u2209)', Operator),
# there_exists / not there_exists / for_all / and / or
(u'(\u2203|\u2204|\u2200|\u2227|\u2228|\u22BB|\223C)',
Operator),
# regex in slot or as string constraint
(r'(\{)(\s*/[^}]+/\s*)(\})',
bygroups(Punctuation, String.Regex, Punctuation)),
# regex in slot or as string constraint
(r'(\{)(\s*\^[^}]+\^\s*)(\})',
bygroups(Punctuation, String.Regex, Punctuation)),
(r'/', Punctuation, 'path'),
# for cardinality etc
(r'(\{)((?:\d+\.\.)?(?:\d+|\*))'
r'((?:\s*;\s*(?:ordered|unordered|unique)){,2})(\})',
bygroups(Punctuation, Number, Number, Punctuation)),
# [{ is start of a tuple value
(r'\[\{', Punctuation),
(r'\}\]', Punctuation),
(r'\{', Punctuation),
(r'\}', Punctuation),
include('constraint_values'),
# type name
(r'[A-Z]\w+(<[A-Z]\w+([A-Za-z_<>]*)>)?', Name.Class),
# attribute name
(r'[a-z_]\w*', Name.Class),
(r'\[', Punctuation, 'any_code'),
(r'(~|//|\\\\|\+|-|/|\*|\^|!=|=|<=|>=|<|>]?)', Operator),
(r'\(', Punctuation),
(r'\)', Punctuation),
# for lists of values
(r',', Punctuation),
(r'"', String, 'string'),
# for assumed value
(r';', Punctuation),
],
}
class AdlLexer(AtomsLexer):
"""
Lexer for ADL syntax.
.. versionadded:: 2.1
"""
name = 'ADL'
aliases = ['adl']
filenames = ['*.adl', '*.adls', '*.adlf', '*.adlx']
tokens = {
'whitespace': [
# blank line ends
(r'\s*\n', Text),
# comment-only line
(r'^[ \t]*--.*$', Comment),
],
'odin_section': [
# repeating the following two rules from the root state enable multi-line
# strings that start in the first column to be dealt with
(r'^(language|description|ontology|terminology|annotations|'
r'component_terminologies|revision_history)[ \t]*\n', Generic.Heading),
(r'^(definition)[ \t]*\n', Generic.Heading, 'cadl_section'),
(r'^([ \t]*|[ \t]+.*)\n', using(OdinLexer)),
(r'^([^"]*")(>[ \t]*\n)', bygroups(String, Punctuation)),
# template overlay delimiter
(r'^----------*\n', Text, '#pop'),
(r'^.*\n', String),
default('#pop'),
],
'cadl_section': [
(r'^([ \t]*|[ \t]+.*)\n', using(CadlLexer)),
default('#pop'),
],
'rules_section': [
(r'^[ \t]+.*\n', using(CadlLexer)),
default('#pop'),
],
'metadata': [
(r'\)', Punctuation, '#pop'),
(r';', Punctuation),
(r'([Tt]rue|[Ff]alse)', Literal),
# numbers and version ids
(r'\d+(\.\d+)*', Literal),
# Guids
(r'(\d|[a-fA-F])+(-(\d|[a-fA-F])+){3,}', Literal),
(r'\w+', Name.Class),
(r'"', String, 'string'),
(r'=', Operator),
(r'[ \t]+', Text),
default('#pop'),
],
'root': [
(r'^(archetype|template_overlay|operational_template|template|'
r'speciali[sz]e)', Generic.Heading),
(r'^(language|description|ontology|terminology|annotations|'
r'component_terminologies|revision_history)[ \t]*\n',
Generic.Heading, 'odin_section'),
(r'^(definition)[ \t]*\n', Generic.Heading, 'cadl_section'),
(r'^(rules)[ \t]*\n', Generic.Heading, 'rules_section'),
include('archetype_id'),
(r'[ \t]*\(', Punctuation, 'metadata'),
include('whitespace'),
],
}

View File

@ -1,641 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexers.asm
~~~~~~~~~~~~~~~~~~~
Lexers for assembly languages.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import re
from pygments.lexer import RegexLexer, include, bygroups, using, words, \
DelegatingLexer
from pygments.lexers.c_cpp import CppLexer, CLexer
from pygments.lexers.d import DLexer
from pygments.token import Text, Name, Number, String, Comment, Punctuation, \
Other, Keyword, Operator
__all__ = ['GasLexer', 'ObjdumpLexer', 'DObjdumpLexer', 'CppObjdumpLexer',
'CObjdumpLexer', 'HsailLexer', 'LlvmLexer', 'NasmLexer',
'NasmObjdumpLexer', 'TasmLexer', 'Ca65Lexer']
class GasLexer(RegexLexer):
"""
For Gas (AT&T) assembly code.
"""
name = 'GAS'
aliases = ['gas', 'asm']
filenames = ['*.s', '*.S']
mimetypes = ['text/x-gas']
#: optional Comment or Whitespace
string = r'"(\\"|[^"])*"'
char = r'[\w$.@-]'
identifier = r'(?:[a-zA-Z$_]' + char + '*|\.' + char + '+)'
number = r'(?:0[xX][a-zA-Z0-9]+|\d+)'
tokens = {
'root': [
include('whitespace'),
(identifier + ':', Name.Label),
(r'\.' + identifier, Name.Attribute, 'directive-args'),
(r'lock|rep(n?z)?|data\d+', Name.Attribute),
(identifier, Name.Function, 'instruction-args'),
(r'[\r\n]+', Text)
],
'directive-args': [
(identifier, Name.Constant),
(string, String),
('@' + identifier, Name.Attribute),
(number, Number.Integer),
(r'[\r\n]+', Text, '#pop'),
include('punctuation'),
include('whitespace')
],
'instruction-args': [
# For objdump-disassembled code, shouldn't occur in
# actual assembler input
('([a-z0-9]+)( )(<)('+identifier+')(>)',
bygroups(Number.Hex, Text, Punctuation, Name.Constant,
Punctuation)),
('([a-z0-9]+)( )(<)('+identifier+')([-+])('+number+')(>)',
bygroups(Number.Hex, Text, Punctuation, Name.Constant,
Punctuation, Number.Integer, Punctuation)),
# Address constants
(identifier, Name.Constant),
(number, Number.Integer),
# Registers
('%' + identifier, Name.Variable),
# Numeric constants
('$'+number, Number.Integer),
(r"$'(.|\\')'", String.Char),
(r'[\r\n]+', Text, '#pop'),
include('punctuation'),
include('whitespace')
],
'whitespace': [
(r'\n', Text),
(r'\s+', Text),
(r'[;#].*?\n', Comment)
],
'punctuation': [
(r'[-*,.()\[\]!:]+', Punctuation)
]
}
def analyse_text(text):
if re.match(r'^\.(text|data|section)', text, re.M):
return True
elif re.match(r'^\.\w+', text, re.M):
return 0.1
def _objdump_lexer_tokens(asm_lexer):
"""
Common objdump lexer tokens to wrap an ASM lexer.
"""
hex_re = r'[0-9A-Za-z]'
return {
'root': [
# File name & format:
('(.*?)(:)( +file format )(.*?)$',
bygroups(Name.Label, Punctuation, Text, String)),
# Section header
('(Disassembly of section )(.*?)(:)$',
bygroups(Text, Name.Label, Punctuation)),
# Function labels
# (With offset)
('('+hex_re+'+)( )(<)(.*?)([-+])(0[xX][A-Za-z0-9]+)(>:)$',
bygroups(Number.Hex, Text, Punctuation, Name.Function,
Punctuation, Number.Hex, Punctuation)),
# (Without offset)
('('+hex_re+'+)( )(<)(.*?)(>:)$',
bygroups(Number.Hex, Text, Punctuation, Name.Function,
Punctuation)),
# Code line with disassembled instructions
('( *)('+hex_re+r'+:)(\t)((?:'+hex_re+hex_re+' )+)( *\t)([a-zA-Z].*?)$',
bygroups(Text, Name.Label, Text, Number.Hex, Text,
using(asm_lexer))),
# Code line with ascii
('( *)('+hex_re+r'+:)(\t)((?:'+hex_re+hex_re+' )+)( *)(.*?)$',
bygroups(Text, Name.Label, Text, Number.Hex, Text, String)),
# Continued code line, only raw opcodes without disassembled
# instruction
('( *)('+hex_re+r'+:)(\t)((?:'+hex_re+hex_re+' )+)$',
bygroups(Text, Name.Label, Text, Number.Hex)),
# Skipped a few bytes
(r'\t\.\.\.$', Text),
# Relocation line
# (With offset)
(r'(\t\t\t)('+hex_re+r'+:)( )([^\t]+)(\t)(.*?)([-+])(0x'+hex_re+'+)$',
bygroups(Text, Name.Label, Text, Name.Property, Text,
Name.Constant, Punctuation, Number.Hex)),
# (Without offset)
(r'(\t\t\t)('+hex_re+r'+:)( )([^\t]+)(\t)(.*?)$',
bygroups(Text, Name.Label, Text, Name.Property, Text,
Name.Constant)),
(r'[^\n]+\n', Other)
]
}
class ObjdumpLexer(RegexLexer):
"""
For the output of 'objdump -dr'
"""
name = 'objdump'
aliases = ['objdump']
filenames = ['*.objdump']
mimetypes = ['text/x-objdump']
tokens = _objdump_lexer_tokens(GasLexer)
class DObjdumpLexer(DelegatingLexer):
"""
For the output of 'objdump -Sr on compiled D files'
"""
name = 'd-objdump'
aliases = ['d-objdump']
filenames = ['*.d-objdump']
mimetypes = ['text/x-d-objdump']
def __init__(self, **options):
super(DObjdumpLexer, self).__init__(DLexer, ObjdumpLexer, **options)
class CppObjdumpLexer(DelegatingLexer):
"""
For the output of 'objdump -Sr on compiled C++ files'
"""
name = 'cpp-objdump'
aliases = ['cpp-objdump', 'c++-objdumb', 'cxx-objdump']
filenames = ['*.cpp-objdump', '*.c++-objdump', '*.cxx-objdump']
mimetypes = ['text/x-cpp-objdump']
def __init__(self, **options):
super(CppObjdumpLexer, self).__init__(CppLexer, ObjdumpLexer, **options)
class CObjdumpLexer(DelegatingLexer):
"""
For the output of 'objdump -Sr on compiled C files'
"""
name = 'c-objdump'
aliases = ['c-objdump']
filenames = ['*.c-objdump']
mimetypes = ['text/x-c-objdump']
def __init__(self, **options):
super(CObjdumpLexer, self).__init__(CLexer, ObjdumpLexer, **options)
class HsailLexer(RegexLexer):
"""
For HSAIL assembly code.
.. versionadded:: 2.2
"""
name = 'HSAIL'
aliases = ['hsail', 'hsa']
filenames = ['*.hsail']
mimetypes = ['text/x-hsail']
string = r'"[^"]*?"'
identifier = r'[a-zA-Z_][\w.]*'
# Registers
register_number = r'[0-9]+'
register = r'(\$(c|s|d|q)' + register_number + ')'
# Qualifiers
alignQual = r'(align\(\d+\))'
widthQual = r'(width\((\d+|all)\))'
allocQual = r'(alloc\(agent\))'
# Instruction Modifiers
roundingMod = (r'((_ftz)?(_up|_down|_zero|_near))')
datatypeMod = (r'_('
# packedTypes
r'u8x4|s8x4|u16x2|s16x2|u8x8|s8x8|u16x4|s16x4|u32x2|s32x2|'
r'u8x16|s8x16|u16x8|s16x8|u32x4|s32x4|u64x2|s64x2|'
r'f16x2|f16x4|f16x8|f32x2|f32x4|f64x2|'
# baseTypes
r'u8|s8|u16|s16|u32|s32|u64|s64|'
r'b128|b8|b16|b32|b64|b1|'
r'f16|f32|f64|'
# opaqueType
r'roimg|woimg|rwimg|samp|sig32|sig64)')
# Numeric Constant
float = r'((\d+\.)|(\d*\.\d+))[eE][+-]?\d+'
hexfloat = r'0[xX](([0-9a-fA-F]+\.[0-9a-fA-F]*)|([0-9a-fA-F]*\.[0-9a-fA-F]+))[pP][+-]?\d+'
ieeefloat = r'0((h|H)[0-9a-fA-F]{4}|(f|F)[0-9a-fA-F]{8}|(d|D)[0-9a-fA-F]{16})'
tokens = {
'root': [
include('whitespace'),
include('comments'),
(string, String),
(r'@' + identifier + ':?', Name.Label),
(register, Name.Variable.Anonymous),
include('keyword'),
(r'&' + identifier, Name.Variable.Global),
(r'%' + identifier, Name.Variable),
(hexfloat, Number.Hex),
(r'0[xX][a-fA-F0-9]+', Number.Hex),
(ieeefloat, Number.Float),
(float, Number.Float),
('\d+', Number.Integer),
(r'[=<>{}\[\]()*.,:;!]|x\b', Punctuation)
],
'whitespace': [
(r'(\n|\s)+', Text),
],
'comments': [
(r'/\*.*?\*/', Comment.Multiline),
(r'//.*?\n', Comment.Single),
],
'keyword': [
# Types
(r'kernarg' + datatypeMod, Keyword.Type),
# Regular keywords
(r'\$(full|base|small|large|default|zero|near)', Keyword),
(words((
'module', 'extension', 'pragma', 'prog', 'indirect', 'signature',
'decl', 'kernel', 'function', 'enablebreakexceptions',
'enabledetectexceptions', 'maxdynamicgroupsize', 'maxflatgridsize',
'maxflatworkgroupsize', 'requireddim', 'requiredgridsize',
'requiredworkgroupsize', 'requirenopartialworkgroups'),
suffix=r'\b'), Keyword),
# instructions
(roundingMod, Keyword),
(datatypeMod, Keyword),
(r'_(' + alignQual + '|' + widthQual + ')', Keyword),
(r'_kernarg', Keyword),
(r'(nop|imagefence)\b', Keyword),
(words((
'cleardetectexcept', 'clock', 'cuid', 'debugtrap', 'dim',
'getdetectexcept', 'groupbaseptr', 'kernargbaseptr', 'laneid',
'maxcuid', 'maxwaveid', 'packetid', 'setdetectexcept', 'waveid',
'workitemflatabsid', 'workitemflatid', 'nullptr', 'abs', 'bitrev',
'currentworkgroupsize', 'currentworkitemflatid', 'fract', 'ncos',
'neg', 'nexp2', 'nlog2', 'nrcp', 'nrsqrt', 'nsin', 'nsqrt',
'gridgroups', 'gridsize', 'not', 'sqrt', 'workgroupid',
'workgroupsize', 'workitemabsid', 'workitemid', 'ceil', 'floor',
'rint', 'trunc', 'add', 'bitmask', 'borrow', 'carry', 'copysign',
'div', 'rem', 'sub', 'shl', 'shr', 'and', 'or', 'xor', 'unpackhi',
'unpacklo', 'max', 'min', 'fma', 'mad', 'bitextract', 'bitselect',
'shuffle', 'cmov', 'bitalign', 'bytealign', 'lerp', 'nfma', 'mul',
'mulhi', 'mul24hi', 'mul24', 'mad24', 'mad24hi', 'bitinsert',
'combine', 'expand', 'lda', 'mov', 'pack', 'unpack', 'packcvt',
'unpackcvt', 'sad', 'sementp', 'ftos', 'stof', 'cmp', 'ld', 'st',
'_eq', '_ne', '_lt', '_le', '_gt', '_ge', '_equ', '_neu', '_ltu',
'_leu', '_gtu', '_geu', '_num', '_nan', '_seq', '_sne', '_slt',
'_sle', '_sgt', '_sge', '_snum', '_snan', '_sequ', '_sneu', '_sltu',
'_sleu', '_sgtu', '_sgeu', 'atomic', '_ld', '_st', '_cas', '_add',
'_and', '_exch', '_max', '_min', '_or', '_sub', '_wrapdec',
'_wrapinc', '_xor', 'ret', 'cvt', '_readonly', '_kernarg', '_global',
'br', 'cbr', 'sbr', '_scacq', '_screl', '_scar', '_rlx', '_wave',
'_wg', '_agent', '_system', 'ldimage', 'stimage', '_v2', '_v3', '_v4',
'_1d', '_2d', '_3d', '_1da', '_2da', '_1db', '_2ddepth', '_2dadepth',
'_width', '_height', '_depth', '_array', '_channelorder',
'_channeltype', 'querysampler', '_coord', '_filter', '_addressing',
'barrier', 'wavebarrier', 'initfbar', 'joinfbar', 'waitfbar',
'arrivefbar', 'leavefbar', 'releasefbar', 'ldf', 'activelaneid',
'activelanecount', 'activelanemask', 'activelanepermute', 'call',
'scall', 'icall', 'alloca', 'packetcompletionsig',
'addqueuewriteindex', 'casqueuewriteindex', 'ldqueuereadindex',
'stqueuereadindex', 'readonly', 'global', 'private', 'group',
'spill', 'arg', '_upi', '_downi', '_zeroi', '_neari', '_upi_sat',
'_downi_sat', '_zeroi_sat', '_neari_sat', '_supi', '_sdowni',
'_szeroi', '_sneari', '_supi_sat', '_sdowni_sat', '_szeroi_sat',
'_sneari_sat', '_pp', '_ps', '_sp', '_ss', '_s', '_p', '_pp_sat',
'_ps_sat', '_sp_sat', '_ss_sat', '_s_sat', '_p_sat')), Keyword),
# Integer types
(r'i[1-9]\d*', Keyword)
]
}
class LlvmLexer(RegexLexer):
"""
For LLVM assembly code.
"""
name = 'LLVM'
aliases = ['llvm']
filenames = ['*.ll']
mimetypes = ['text/x-llvm']
#: optional Comment or Whitespace
string = r'"[^"]*?"'
identifier = r'([-a-zA-Z$._][\w\-$.]*|' + string + ')'
tokens = {
'root': [
include('whitespace'),
# Before keywords, because keywords are valid label names :(...
(identifier + '\s*:', Name.Label),
include('keyword'),
(r'%' + identifier, Name.Variable),
(r'@' + identifier, Name.Variable.Global),
(r'%\d+', Name.Variable.Anonymous),
(r'@\d+', Name.Variable.Global),
(r'#\d+', Name.Variable.Global),
(r'!' + identifier, Name.Variable),
(r'!\d+', Name.Variable.Anonymous),
(r'c?' + string, String),
(r'0[xX][a-fA-F0-9]+', Number),
(r'-?\d+(?:[.]\d+)?(?:[eE][-+]?\d+(?:[.]\d+)?)?', Number),
(r'[=<>{}\[\]()*.,!]|x\b', Punctuation)
],
'whitespace': [
(r'(\n|\s)+', Text),
(r';.*?\n', Comment)
],
'keyword': [
# Regular keywords
(words((
'begin', 'end', 'true', 'false', 'declare', 'define', 'global',
'constant', 'private', 'linker_private', 'internal',
'available_externally', 'linkonce', 'linkonce_odr', 'weak',
'weak_odr', 'appending', 'dllimport', 'dllexport', 'common',
'default', 'hidden', 'protected', 'extern_weak', 'external',
'thread_local', 'zeroinitializer', 'undef', 'null', 'to', 'tail',
'target', 'triple', 'datalayout', 'volatile', 'nuw', 'nsw', 'nnan',
'ninf', 'nsz', 'arcp', 'fast', 'exact', 'inbounds', 'align',
'addrspace', 'section', 'alias', 'module', 'asm', 'sideeffect',
'gc', 'dbg', 'linker_private_weak', 'attributes', 'blockaddress',
'initialexec', 'localdynamic', 'localexec', 'prefix', 'unnamed_addr',
'ccc', 'fastcc', 'coldcc', 'x86_stdcallcc', 'x86_fastcallcc',
'arm_apcscc', 'arm_aapcscc', 'arm_aapcs_vfpcc', 'ptx_device',
'ptx_kernel', 'intel_ocl_bicc', 'msp430_intrcc', 'spir_func',
'spir_kernel', 'x86_64_sysvcc', 'x86_64_win64cc', 'x86_thiscallcc',
'cc', 'c', 'signext', 'zeroext', 'inreg', 'sret', 'nounwind',
'noreturn', 'noalias', 'nocapture', 'byval', 'nest', 'readnone',
'readonly', 'inlinehint', 'noinline', 'alwaysinline', 'optsize', 'ssp',
'sspreq', 'noredzone', 'noimplicitfloat', 'naked', 'builtin', 'cold',
'nobuiltin', 'noduplicate', 'nonlazybind', 'optnone', 'returns_twice',
'sanitize_address', 'sanitize_memory', 'sanitize_thread', 'sspstrong',
'uwtable', 'returned', 'type', 'opaque', 'eq', 'ne', 'slt', 'sgt',
'sle', 'sge', 'ult', 'ugt', 'ule', 'uge', 'oeq', 'one', 'olt', 'ogt',
'ole', 'oge', 'ord', 'uno', 'ueq', 'une', 'x', 'acq_rel', 'acquire',
'alignstack', 'atomic', 'catch', 'cleanup', 'filter', 'inteldialect',
'max', 'min', 'monotonic', 'nand', 'personality', 'release', 'seq_cst',
'singlethread', 'umax', 'umin', 'unordered', 'xchg', 'add', 'fadd',
'sub', 'fsub', 'mul', 'fmul', 'udiv', 'sdiv', 'fdiv', 'urem', 'srem',
'frem', 'shl', 'lshr', 'ashr', 'and', 'or', 'xor', 'icmp', 'fcmp',
'phi', 'call', 'trunc', 'zext', 'sext', 'fptrunc', 'fpext', 'uitofp',
'sitofp', 'fptoui', 'fptosi', 'inttoptr', 'ptrtoint', 'bitcast',
'addrspacecast', 'select', 'va_arg', 'ret', 'br', 'switch', 'invoke',
'unwind', 'unreachable', 'indirectbr', 'landingpad', 'resume',
'malloc', 'alloca', 'free', 'load', 'store', 'getelementptr',
'extractelement', 'insertelement', 'shufflevector', 'getresult',
'extractvalue', 'insertvalue', 'atomicrmw', 'cmpxchg', 'fence',
'allocsize', 'amdgpu_cs', 'amdgpu_gs', 'amdgpu_kernel', 'amdgpu_ps',
'amdgpu_vs', 'any', 'anyregcc', 'argmemonly', 'avr_intrcc',
'avr_signalcc', 'caller', 'catchpad', 'catchret', 'catchswitch',
'cleanuppad', 'cleanupret', 'comdat', 'convergent', 'cxx_fast_tlscc',
'deplibs', 'dereferenceable', 'dereferenceable_or_null', 'distinct',
'exactmatch', 'externally_initialized', 'from', 'ghccc', 'hhvm_ccc',
'hhvmcc', 'ifunc', 'inaccessiblemem_or_argmemonly', 'inaccessiblememonly',
'inalloca', 'jumptable', 'largest', 'local_unnamed_addr', 'minsize',
'musttail', 'noduplicates', 'none', 'nonnull', 'norecurse', 'notail',
'preserve_allcc', 'preserve_mostcc', 'prologue', 'safestack', 'samesize',
'source_filename', 'swiftcc', 'swifterror', 'swiftself', 'webkit_jscc',
'within', 'writeonly', 'x86_intrcc', 'x86_vectorcallcc'),
suffix=r'\b'), Keyword),
# Types
(words(('void', 'half', 'float', 'double', 'x86_fp80', 'fp128',
'ppc_fp128', 'label', 'metadata', 'token')), Keyword.Type),
# Integer types
(r'i[1-9]\d*', Keyword)
]
}
class NasmLexer(RegexLexer):
"""
For Nasm (Intel) assembly code.
"""
name = 'NASM'
aliases = ['nasm']
filenames = ['*.asm', '*.ASM']
mimetypes = ['text/x-nasm']
identifier = r'[a-z$._?][\w$.?#@~]*'
hexn = r'(?:0x[0-9a-f]+|$0[0-9a-f]*|[0-9]+[0-9a-f]*h)'
octn = r'[0-7]+q'
binn = r'[01]+b'
decn = r'[0-9]+'
floatn = decn + r'\.e?' + decn
string = r'"(\\"|[^"\n])*"|' + r"'(\\'|[^'\n])*'|" + r"`(\\`|[^`\n])*`"
declkw = r'(?:res|d)[bwdqt]|times'
register = (r'r[0-9][0-5]?[bwd]|'
r'[a-d][lh]|[er]?[a-d]x|[er]?[sb]p|[er]?[sd]i|[c-gs]s|st[0-7]|'
r'mm[0-7]|cr[0-4]|dr[0-367]|tr[3-7]')
wordop = r'seg|wrt|strict'
type = r'byte|[dq]?word'
directives = (r'BITS|USE16|USE32|SECTION|SEGMENT|ABSOLUTE|EXTERN|GLOBAL|'
r'ORG|ALIGN|STRUC|ENDSTRUC|COMMON|CPU|GROUP|UPPERCASE|IMPORT|'
r'EXPORT|LIBRARY|MODULE')
flags = re.IGNORECASE | re.MULTILINE
tokens = {
'root': [
(r'^\s*%', Comment.Preproc, 'preproc'),
include('whitespace'),
(identifier + ':', Name.Label),
(r'(%s)(\s+)(equ)' % identifier,
bygroups(Name.Constant, Keyword.Declaration, Keyword.Declaration),
'instruction-args'),
(directives, Keyword, 'instruction-args'),
(declkw, Keyword.Declaration, 'instruction-args'),
(identifier, Name.Function, 'instruction-args'),
(r'[\r\n]+', Text)
],
'instruction-args': [
(string, String),
(hexn, Number.Hex),
(octn, Number.Oct),
(binn, Number.Bin),
(floatn, Number.Float),
(decn, Number.Integer),
include('punctuation'),
(register, Name.Builtin),
(identifier, Name.Variable),
(r'[\r\n]+', Text, '#pop'),
include('whitespace')
],
'preproc': [
(r'[^;\n]+', Comment.Preproc),
(r';.*?\n', Comment.Single, '#pop'),
(r'\n', Comment.Preproc, '#pop'),
],
'whitespace': [
(r'\n', Text),
(r'[ \t]+', Text),
(r';.*', Comment.Single)
],
'punctuation': [
(r'[,():\[\]]+', Punctuation),
(r'[&|^<>+*/%~-]+', Operator),
(r'[$]+', Keyword.Constant),
(wordop, Operator.Word),
(type, Keyword.Type)
],
}
class NasmObjdumpLexer(ObjdumpLexer):
"""
For the output of 'objdump -d -M intel'.
.. versionadded:: 2.0
"""
name = 'objdump-nasm'
aliases = ['objdump-nasm']
filenames = ['*.objdump-intel']
mimetypes = ['text/x-nasm-objdump']
tokens = _objdump_lexer_tokens(NasmLexer)
class TasmLexer(RegexLexer):
"""
For Tasm (Turbo Assembler) assembly code.
"""
name = 'TASM'
aliases = ['tasm']
filenames = ['*.asm', '*.ASM', '*.tasm']
mimetypes = ['text/x-tasm']
identifier = r'[@a-z$._?][\w$.?#@~]*'
hexn = r'(?:0x[0-9a-f]+|$0[0-9a-f]*|[0-9]+[0-9a-f]*h)'
octn = r'[0-7]+q'
binn = r'[01]+b'
decn = r'[0-9]+'
floatn = decn + r'\.e?' + decn
string = r'"(\\"|[^"\n])*"|' + r"'(\\'|[^'\n])*'|" + r"`(\\`|[^`\n])*`"
declkw = r'(?:res|d)[bwdqt]|times'
register = (r'r[0-9][0-5]?[bwd]|'
r'[a-d][lh]|[er]?[a-d]x|[er]?[sb]p|[er]?[sd]i|[c-gs]s|st[0-7]|'
r'mm[0-7]|cr[0-4]|dr[0-367]|tr[3-7]')
wordop = r'seg|wrt|strict'
type = r'byte|[dq]?word'
directives = (r'BITS|USE16|USE32|SECTION|SEGMENT|ABSOLUTE|EXTERN|GLOBAL|'
r'ORG|ALIGN|STRUC|ENDSTRUC|ENDS|COMMON|CPU|GROUP|UPPERCASE|INCLUDE|'
r'EXPORT|LIBRARY|MODULE|PROC|ENDP|USES|ARG|DATASEG|UDATASEG|END|IDEAL|'
r'P386|MODEL|ASSUME|CODESEG|SIZE')
# T[A-Z][a-z] is more of a convention. Lexer should filter out STRUC definitions
# and then 'add' them to datatype somehow.
datatype = (r'db|dd|dw|T[A-Z][a-z]+')
flags = re.IGNORECASE | re.MULTILINE
tokens = {
'root': [
(r'^\s*%', Comment.Preproc, 'preproc'),
include('whitespace'),
(identifier + ':', Name.Label),
(directives, Keyword, 'instruction-args'),
(r'(%s)(\s+)(%s)' % (identifier, datatype),
bygroups(Name.Constant, Keyword.Declaration, Keyword.Declaration),
'instruction-args'),
(declkw, Keyword.Declaration, 'instruction-args'),
(identifier, Name.Function, 'instruction-args'),
(r'[\r\n]+', Text)
],
'instruction-args': [
(string, String),
(hexn, Number.Hex),
(octn, Number.Oct),
(binn, Number.Bin),
(floatn, Number.Float),
(decn, Number.Integer),
include('punctuation'),
(register, Name.Builtin),
(identifier, Name.Variable),
# Do not match newline when it's preceeded by a backslash
(r'(\\\s*)(;.*)([\r\n])', bygroups(Text, Comment.Single, Text)),
(r'[\r\n]+', Text, '#pop'),
include('whitespace')
],
'preproc': [
(r'[^;\n]+', Comment.Preproc),
(r';.*?\n', Comment.Single, '#pop'),
(r'\n', Comment.Preproc, '#pop'),
],
'whitespace': [
(r'[\n\r]', Text),
(r'\\[\n\r]', Text),
(r'[ \t]+', Text),
(r';.*', Comment.Single)
],
'punctuation': [
(r'[,():\[\]]+', Punctuation),
(r'[&|^<>+*=/%~-]+', Operator),
(r'[$]+', Keyword.Constant),
(wordop, Operator.Word),
(type, Keyword.Type)
],
}
class Ca65Lexer(RegexLexer):
"""
For ca65 assembler sources.
.. versionadded:: 1.6
"""
name = 'ca65 assembler'
aliases = ['ca65']
filenames = ['*.s']
flags = re.IGNORECASE
tokens = {
'root': [
(r';.*', Comment.Single),
(r'\s+', Text),
(r'[a-z_.@$][\w.@$]*:', Name.Label),
(r'((ld|st)[axy]|(in|de)[cxy]|asl|lsr|ro[lr]|adc|sbc|cmp|cp[xy]'
r'|cl[cvdi]|se[cdi]|jmp|jsr|bne|beq|bpl|bmi|bvc|bvs|bcc|bcs'
r'|p[lh][ap]|rt[is]|brk|nop|ta[xy]|t[xy]a|txs|tsx|and|ora|eor'
r'|bit)\b', Keyword),
(r'\.\w+', Keyword.Pseudo),
(r'[-+~*/^&|!<>=]', Operator),
(r'"[^"\n]*.', String),
(r"'[^'\n]*.", String.Char),
(r'\$[0-9a-f]+|[0-9a-f]+h\b', Number.Hex),
(r'\d+', Number.Integer),
(r'%[01]+', Number.Bin),
(r'[#,.:()=\[\]]', Punctuation),
(r'[a-z_.@$][\w.@$]*', Name),
]
}
def analyse_text(self, text):
# comments in GAS start with "#"
if re.match(r'^\s*;', text, re.MULTILINE):
return 0.9

View File

@ -1,374 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexers.automation
~~~~~~~~~~~~~~~~~~~~~~~~~~
Lexers for automation scripting languages.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
from pygments.lexer import RegexLexer, include, bygroups, combined
from pygments.token import Text, Comment, Operator, Name, String, \
Number, Punctuation, Generic
__all__ = ['AutohotkeyLexer', 'AutoItLexer']
class AutohotkeyLexer(RegexLexer):
"""
For `autohotkey <http://www.autohotkey.com/>`_ source code.
.. versionadded:: 1.4
"""
name = 'autohotkey'
aliases = ['ahk', 'autohotkey']
filenames = ['*.ahk', '*.ahkl']
mimetypes = ['text/x-autohotkey']
tokens = {
'root': [
(r'^(\s*)(/\*)', bygroups(Text, Comment.Multiline), 'incomment'),
(r'^(\s*)(\()', bygroups(Text, Generic), 'incontinuation'),
(r'\s+;.*?$', Comment.Single),
(r'^;.*?$', Comment.Single),
(r'[]{}(),;[]', Punctuation),
(r'(in|is|and|or|not)\b', Operator.Word),
(r'\%[a-zA-Z_#@$][\w#@$]*\%', Name.Variable),
(r'!=|==|:=|\.=|<<|>>|[-~+/*%=<>&^|?:!.]', Operator),
include('commands'),
include('labels'),
include('builtInFunctions'),
include('builtInVariables'),
(r'"', String, combined('stringescape', 'dqs')),
include('numbers'),
(r'[a-zA-Z_#@$][\w#@$]*', Name),
(r'\\|\'', Text),
(r'\`([,%`abfnrtv\-+;])', String.Escape),
include('garbage'),
],
'incomment': [
(r'^\s*\*/', Comment.Multiline, '#pop'),
(r'[^*/]', Comment.Multiline),
(r'[*/]', Comment.Multiline)
],
'incontinuation': [
(r'^\s*\)', Generic, '#pop'),
(r'[^)]', Generic),
(r'[)]', Generic),
],
'commands': [
(r'(?i)^(\s*)(global|local|static|'
r'#AllowSameLineComments|#ClipboardTimeout|#CommentFlag|'
r'#ErrorStdOut|#EscapeChar|#HotkeyInterval|#HotkeyModifierTimeout|'
r'#Hotstring|#IfWinActive|#IfWinExist|#IfWinNotActive|'
r'#IfWinNotExist|#IncludeAgain|#Include|#InstallKeybdHook|'
r'#InstallMouseHook|#KeyHistory|#LTrim|#MaxHotkeysPerInterval|'
r'#MaxMem|#MaxThreads|#MaxThreadsBuffer|#MaxThreadsPerHotkey|'
r'#NoEnv|#NoTrayIcon|#Persistent|#SingleInstance|#UseHook|'
r'#WinActivateForce|AutoTrim|BlockInput|Break|Click|ClipWait|'
r'Continue|Control|ControlClick|ControlFocus|ControlGetFocus|'
r'ControlGetPos|ControlGetText|ControlGet|ControlMove|ControlSend|'
r'ControlSendRaw|ControlSetText|CoordMode|Critical|'
r'DetectHiddenText|DetectHiddenWindows|Drive|DriveGet|'
r'DriveSpaceFree|Edit|Else|EnvAdd|EnvDiv|EnvGet|EnvMult|EnvSet|'
r'EnvSub|EnvUpdate|Exit|ExitApp|FileAppend|'
r'FileCopy|FileCopyDir|FileCreateDir|FileCreateShortcut|'
r'FileDelete|FileGetAttrib|FileGetShortcut|FileGetSize|'
r'FileGetTime|FileGetVersion|FileInstall|FileMove|FileMoveDir|'
r'FileRead|FileReadLine|FileRecycle|FileRecycleEmpty|'
r'FileRemoveDir|FileSelectFile|FileSelectFolder|FileSetAttrib|'
r'FileSetTime|FormatTime|GetKeyState|Gosub|Goto|GroupActivate|'
r'GroupAdd|GroupClose|GroupDeactivate|Gui|GuiControl|'
r'GuiControlGet|Hotkey|IfEqual|IfExist|IfGreaterOrEqual|IfGreater|'
r'IfInString|IfLess|IfLessOrEqual|IfMsgBox|IfNotEqual|IfNotExist|'
r'IfNotInString|IfWinActive|IfWinExist|IfWinNotActive|'
r'IfWinNotExist|If |ImageSearch|IniDelete|IniRead|IniWrite|'
r'InputBox|Input|KeyHistory|KeyWait|ListHotkeys|ListLines|'
r'ListVars|Loop|Menu|MouseClickDrag|MouseClick|MouseGetPos|'
r'MouseMove|MsgBox|OnExit|OutputDebug|Pause|PixelGetColor|'
r'PixelSearch|PostMessage|Process|Progress|Random|RegDelete|'
r'RegRead|RegWrite|Reload|Repeat|Return|RunAs|RunWait|Run|'
r'SendEvent|SendInput|SendMessage|SendMode|SendPlay|SendRaw|Send|'
r'SetBatchLines|SetCapslockState|SetControlDelay|'
r'SetDefaultMouseSpeed|SetEnv|SetFormat|SetKeyDelay|'
r'SetMouseDelay|SetNumlockState|SetScrollLockState|'
r'SetStoreCapslockMode|SetTimer|SetTitleMatchMode|'
r'SetWinDelay|SetWorkingDir|Shutdown|Sleep|Sort|SoundBeep|'
r'SoundGet|SoundGetWaveVolume|SoundPlay|SoundSet|'
r'SoundSetWaveVolume|SplashImage|SplashTextOff|SplashTextOn|'
r'SplitPath|StatusBarGetText|StatusBarWait|StringCaseSense|'
r'StringGetPos|StringLeft|StringLen|StringLower|StringMid|'
r'StringReplace|StringRight|StringSplit|StringTrimLeft|'
r'StringTrimRight|StringUpper|Suspend|SysGet|Thread|ToolTip|'
r'Transform|TrayTip|URLDownloadToFile|While|WinActivate|'
r'WinActivateBottom|WinClose|WinGetActiveStats|WinGetActiveTitle|'
r'WinGetClass|WinGetPos|WinGetText|WinGetTitle|WinGet|WinHide|'
r'WinKill|WinMaximize|WinMenuSelectItem|WinMinimizeAllUndo|'
r'WinMinimizeAll|WinMinimize|WinMove|WinRestore|WinSetTitle|'
r'WinSet|WinShow|WinWaitActive|WinWaitClose|WinWaitNotActive|'
r'WinWait)\b', bygroups(Text, Name.Builtin)),
],
'builtInFunctions': [
(r'(?i)(Abs|ACos|Asc|ASin|ATan|Ceil|Chr|Cos|DllCall|Exp|FileExist|'
r'Floor|GetKeyState|IL_Add|IL_Create|IL_Destroy|InStr|IsFunc|'
r'IsLabel|Ln|Log|LV_Add|LV_Delete|LV_DeleteCol|LV_GetCount|'
r'LV_GetNext|LV_GetText|LV_Insert|LV_InsertCol|LV_Modify|'
r'LV_ModifyCol|LV_SetImageList|Mod|NumGet|NumPut|OnMessage|'
r'RegExMatch|RegExReplace|RegisterCallback|Round|SB_SetIcon|'
r'SB_SetParts|SB_SetText|Sin|Sqrt|StrLen|SubStr|Tan|TV_Add|'
r'TV_Delete|TV_GetChild|TV_GetCount|TV_GetNext|TV_Get|'
r'TV_GetParent|TV_GetPrev|TV_GetSelection|TV_GetText|TV_Modify|'
r'VarSetCapacity|WinActive|WinExist|Object|ComObjActive|'
r'ComObjArray|ComObjEnwrap|ComObjUnwrap|ComObjParameter|'
r'ComObjType|ComObjConnect|ComObjCreate|ComObjGet|ComObjError|'
r'ComObjValue|Insert|MinIndex|MaxIndex|Remove|SetCapacity|'
r'GetCapacity|GetAddress|_NewEnum|FileOpen|Read|Write|ReadLine|'
r'WriteLine|ReadNumType|WriteNumType|RawRead|RawWrite|Seek|Tell|'
r'Close|Next|IsObject|StrPut|StrGet|Trim|LTrim|RTrim)\b',
Name.Function),
],
'builtInVariables': [
(r'(?i)(A_AhkPath|A_AhkVersion|A_AppData|A_AppDataCommon|'
r'A_AutoTrim|A_BatchLines|A_CaretX|A_CaretY|A_ComputerName|'
r'A_ControlDelay|A_Cursor|A_DDDD|A_DDD|A_DD|A_DefaultMouseSpeed|'
r'A_Desktop|A_DesktopCommon|A_DetectHiddenText|'
r'A_DetectHiddenWindows|A_EndChar|A_EventInfo|A_ExitReason|'
r'A_FormatFloat|A_FormatInteger|A_Gui|A_GuiEvent|A_GuiControl|'
r'A_GuiControlEvent|A_GuiHeight|A_GuiWidth|A_GuiX|A_GuiY|A_Hour|'
r'A_IconFile|A_IconHidden|A_IconNumber|A_IconTip|A_Index|'
r'A_IPAddress1|A_IPAddress2|A_IPAddress3|A_IPAddress4|A_ISAdmin|'
r'A_IsCompiled|A_IsCritical|A_IsPaused|A_IsSuspended|A_KeyDelay|'
r'A_Language|A_LastError|A_LineFile|A_LineNumber|A_LoopField|'
r'A_LoopFileAttrib|A_LoopFileDir|A_LoopFileExt|A_LoopFileFullPath|'
r'A_LoopFileLongPath|A_LoopFileName|A_LoopFileShortName|'
r'A_LoopFileShortPath|A_LoopFileSize|A_LoopFileSizeKB|'
r'A_LoopFileSizeMB|A_LoopFileTimeAccessed|A_LoopFileTimeCreated|'
r'A_LoopFileTimeModified|A_LoopReadLine|A_LoopRegKey|'
r'A_LoopRegName|A_LoopRegSubkey|A_LoopRegTimeModified|'
r'A_LoopRegType|A_MDAY|A_Min|A_MM|A_MMM|A_MMMM|A_Mon|A_MouseDelay|'
r'A_MSec|A_MyDocuments|A_Now|A_NowUTC|A_NumBatchLines|A_OSType|'
r'A_OSVersion|A_PriorHotkey|A_ProgramFiles|A_Programs|'
r'A_ProgramsCommon|A_ScreenHeight|A_ScreenWidth|A_ScriptDir|'
r'A_ScriptFullPath|A_ScriptName|A_Sec|A_Space|A_StartMenu|'
r'A_StartMenuCommon|A_Startup|A_StartupCommon|A_StringCaseSense|'
r'A_Tab|A_Temp|A_ThisFunc|A_ThisHotkey|A_ThisLabel|A_ThisMenu|'
r'A_ThisMenuItem|A_ThisMenuItemPos|A_TickCount|A_TimeIdle|'
r'A_TimeIdlePhysical|A_TimeSincePriorHotkey|A_TimeSinceThisHotkey|'
r'A_TitleMatchMode|A_TitleMatchModeSpeed|A_UserName|A_WDay|'
r'A_WinDelay|A_WinDir|A_WorkingDir|A_YDay|A_YEAR|A_YWeek|A_YYYY|'
r'Clipboard|ClipboardAll|ComSpec|ErrorLevel|ProgramFiles|True|'
r'False|A_IsUnicode|A_FileEncoding|A_OSVersion|A_PtrSize)\b',
Name.Variable),
],
'labels': [
# hotkeys and labels
# technically, hotkey names are limited to named keys and buttons
(r'(^\s*)([^:\s("]+?:{1,2})', bygroups(Text, Name.Label)),
(r'(^\s*)(::[^:\s]+?::)', bygroups(Text, Name.Label)),
],
'numbers': [
(r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
(r'\d+[eE][+-]?[0-9]+', Number.Float),
(r'0\d+', Number.Oct),
(r'0[xX][a-fA-F0-9]+', Number.Hex),
(r'\d+L', Number.Integer.Long),
(r'\d+', Number.Integer)
],
'stringescape': [
(r'\"\"|\`([,%`abfnrtv])', String.Escape),
],
'strings': [
(r'[^"\n]+', String),
],
'dqs': [
(r'"', String, '#pop'),
include('strings')
],
'garbage': [
(r'[^\S\n]', Text),
# (r'.', Text), # no cheating
],
}
class AutoItLexer(RegexLexer):
"""
For `AutoIt <http://www.autoitscript.com/site/autoit/>`_ files.
AutoIt is a freeware BASIC-like scripting language
designed for automating the Windows GUI and general scripting
.. versionadded:: 1.6
"""
name = 'AutoIt'
aliases = ['autoit']
filenames = ['*.au3']
mimetypes = ['text/x-autoit']
# Keywords, functions, macros from au3.keywords.properties
# which can be found in AutoIt installed directory, e.g.
# c:\Program Files (x86)\AutoIt3\SciTE\au3.keywords.properties
keywords = """\
#include-once #include #endregion #forcedef #forceref #region
and byref case continueloop dim do else elseif endfunc endif
endselect exit exitloop for func global
if local next not or return select step
then to until wend while exit""".split()
functions = """\
abs acos adlibregister adlibunregister asc ascw asin assign atan
autoitsetoption autoitwingettitle autoitwinsettitle beep binary binarylen
binarymid binarytostring bitand bitnot bitor bitrotate bitshift bitxor
blockinput break call cdtray ceiling chr chrw clipget clipput consoleread
consolewrite consolewriteerror controlclick controlcommand controldisable
controlenable controlfocus controlgetfocus controlgethandle controlgetpos
controlgettext controlhide controllistview controlmove controlsend
controlsettext controlshow controltreeview cos dec dircopy dircreate
dirgetsize dirmove dirremove dllcall dllcalladdress dllcallbackfree
dllcallbackgetptr dllcallbackregister dllclose dllopen dllstructcreate
dllstructgetdata dllstructgetptr dllstructgetsize dllstructsetdata
drivegetdrive drivegetfilesystem drivegetlabel drivegetserial drivegettype
drivemapadd drivemapdel drivemapget drivesetlabel drivespacefree
drivespacetotal drivestatus envget envset envupdate eval execute exp
filechangedir fileclose filecopy filecreatentfslink filecreateshortcut
filedelete fileexists filefindfirstfile filefindnextfile fileflush
filegetattrib filegetencoding filegetlongname filegetpos filegetshortcut
filegetshortname filegetsize filegettime filegetversion fileinstall filemove
fileopen fileopendialog fileread filereadline filerecycle filerecycleempty
filesavedialog fileselectfolder filesetattrib filesetpos filesettime
filewrite filewriteline floor ftpsetproxy guicreate guictrlcreateavi
guictrlcreatebutton guictrlcreatecheckbox guictrlcreatecombo
guictrlcreatecontextmenu guictrlcreatedate guictrlcreatedummy
guictrlcreateedit guictrlcreategraphic guictrlcreategroup guictrlcreateicon
guictrlcreateinput guictrlcreatelabel guictrlcreatelist
guictrlcreatelistview guictrlcreatelistviewitem guictrlcreatemenu
guictrlcreatemenuitem guictrlcreatemonthcal guictrlcreateobj
guictrlcreatepic guictrlcreateprogress guictrlcreateradio
guictrlcreateslider guictrlcreatetab guictrlcreatetabitem
guictrlcreatetreeview guictrlcreatetreeviewitem guictrlcreateupdown
guictrldelete guictrlgethandle guictrlgetstate guictrlread guictrlrecvmsg
guictrlregisterlistviewsort guictrlsendmsg guictrlsendtodummy
guictrlsetbkcolor guictrlsetcolor guictrlsetcursor guictrlsetdata
guictrlsetdefbkcolor guictrlsetdefcolor guictrlsetfont guictrlsetgraphic
guictrlsetimage guictrlsetlimit guictrlsetonevent guictrlsetpos
guictrlsetresizing guictrlsetstate guictrlsetstyle guictrlsettip guidelete
guigetcursorinfo guigetmsg guigetstyle guiregistermsg guisetaccelerators
guisetbkcolor guisetcoord guisetcursor guisetfont guisethelp guiseticon
guisetonevent guisetstate guisetstyle guistartgroup guiswitch hex hotkeyset
httpsetproxy httpsetuseragent hwnd inetclose inetget inetgetinfo inetgetsize
inetread inidelete iniread inireadsection inireadsectionnames
inirenamesection iniwrite iniwritesection inputbox int isadmin isarray
isbinary isbool isdeclared isdllstruct isfloat ishwnd isint iskeyword
isnumber isobj isptr isstring log memgetstats mod mouseclick mouseclickdrag
mousedown mousegetcursor mousegetpos mousemove mouseup mousewheel msgbox
number objcreate objcreateinterface objevent objevent objget objname
onautoitexitregister onautoitexitunregister opt ping pixelchecksum
pixelgetcolor pixelsearch pluginclose pluginopen processclose processexists
processgetstats processlist processsetpriority processwait processwaitclose
progressoff progresson progressset ptr random regdelete regenumkey
regenumval regread regwrite round run runas runaswait runwait send
sendkeepactive seterror setextended shellexecute shellexecutewait shutdown
sin sleep soundplay soundsetwavevolume splashimageon splashoff splashtexton
sqrt srandom statusbargettext stderrread stdinwrite stdioclose stdoutread
string stringaddcr stringcompare stringformat stringfromasciiarray
stringinstr stringisalnum stringisalpha stringisascii stringisdigit
stringisfloat stringisint stringislower stringisspace stringisupper
stringisxdigit stringleft stringlen stringlower stringmid stringregexp
stringregexpreplace stringreplace stringright stringsplit stringstripcr
stringstripws stringtoasciiarray stringtobinary stringtrimleft
stringtrimright stringupper tan tcpaccept tcpclosesocket tcpconnect
tcplisten tcpnametoip tcprecv tcpsend tcpshutdown tcpstartup timerdiff
timerinit tooltip traycreateitem traycreatemenu traygetmsg trayitemdelete
trayitemgethandle trayitemgetstate trayitemgettext trayitemsetonevent
trayitemsetstate trayitemsettext traysetclick trayseticon traysetonevent
traysetpauseicon traysetstate traysettooltip traytip ubound udpbind
udpclosesocket udpopen udprecv udpsend udpshutdown udpstartup vargettype
winactivate winactive winclose winexists winflash wingetcaretpos
wingetclasslist wingetclientsize wingethandle wingetpos wingetprocess
wingetstate wingettext wingettitle winkill winlist winmenuselectitem
winminimizeall winminimizeallundo winmove winsetontop winsetstate
winsettitle winsettrans winwait winwaitactive winwaitclose
winwaitnotactive""".split()
macros = """\
@appdatacommondir @appdatadir @autoitexe @autoitpid @autoitversion
@autoitx64 @com_eventobj @commonfilesdir @compiled @computername @comspec
@cpuarch @cr @crlf @desktopcommondir @desktopdepth @desktopdir
@desktopheight @desktoprefresh @desktopwidth @documentscommondir @error
@exitcode @exitmethod @extended @favoritescommondir @favoritesdir
@gui_ctrlhandle @gui_ctrlid @gui_dragfile @gui_dragid @gui_dropid
@gui_winhandle @homedrive @homepath @homeshare @hotkeypressed @hour
@ipaddress1 @ipaddress2 @ipaddress3 @ipaddress4 @kblayout @lf
@logondnsdomain @logondomain @logonserver @mday @min @mon @msec @muilang
@mydocumentsdir @numparams @osarch @osbuild @oslang @osservicepack @ostype
@osversion @programfilesdir @programscommondir @programsdir @scriptdir
@scriptfullpath @scriptlinenumber @scriptname @sec @startmenucommondir
@startmenudir @startupcommondir @startupdir @sw_disable @sw_enable @sw_hide
@sw_lock @sw_maximize @sw_minimize @sw_restore @sw_show @sw_showdefault
@sw_showmaximized @sw_showminimized @sw_showminnoactive @sw_showna
@sw_shownoactivate @sw_shownormal @sw_unlock @systemdir @tab @tempdir
@tray_id @trayiconflashing @trayiconvisible @username @userprofiledir @wday
@windowsdir @workingdir @yday @year""".split()
tokens = {
'root': [
(r';.*\n', Comment.Single),
(r'(#comments-start|#cs)(.|\n)*?(#comments-end|#ce)',
Comment.Multiline),
(r'[\[\]{}(),;]', Punctuation),
(r'(and|or|not)\b', Operator.Word),
(r'[$|@][a-zA-Z_]\w*', Name.Variable),
(r'!=|==|:=|\.=|<<|>>|[-~+/*%=<>&^|?:!.]', Operator),
include('commands'),
include('labels'),
include('builtInFunctions'),
include('builtInMarcros'),
(r'"', String, combined('stringescape', 'dqs')),
include('numbers'),
(r'[a-zA-Z_#@$][\w#@$]*', Name),
(r'\\|\'', Text),
(r'\`([,%`abfnrtv\-+;])', String.Escape),
(r'_\n', Text), # Line continuation
include('garbage'),
],
'commands': [
(r'(?i)(\s*)(%s)\b' % '|'.join(keywords),
bygroups(Text, Name.Builtin)),
],
'builtInFunctions': [
(r'(?i)(%s)\b' % '|'.join(functions),
Name.Function),
],
'builtInMarcros': [
(r'(?i)(%s)\b' % '|'.join(macros),
Name.Variable.Global),
],
'labels': [
# sendkeys
(r'(^\s*)(\{\S+?\})', bygroups(Text, Name.Label)),
],
'numbers': [
(r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
(r'\d+[eE][+-]?[0-9]+', Number.Float),
(r'0\d+', Number.Oct),
(r'0[xX][a-fA-F0-9]+', Number.Hex),
(r'\d+L', Number.Integer.Long),
(r'\d+', Number.Integer)
],
'stringescape': [
(r'\"\"|\`([,%`abfnrtv])', String.Escape),
],
'strings': [
(r'[^"\n]+', String),
],
'dqs': [
(r'"', String, '#pop'),
include('strings')
],
'garbage': [
(r'[^\S\n]', Text),
],
}

View File

@ -1,500 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexers.basic
~~~~~~~~~~~~~~~~~~~~~
Lexers for BASIC like languages (other than VB.net).
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import re
from pygments.lexer import RegexLexer, bygroups, default, words, include
from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
Number, Punctuation
__all__ = ['BlitzBasicLexer', 'BlitzMaxLexer', 'MonkeyLexer', 'CbmBasicV2Lexer',
'QBasicLexer']
class BlitzMaxLexer(RegexLexer):
"""
For `BlitzMax <http://blitzbasic.com>`_ source code.
.. versionadded:: 1.4
"""
name = 'BlitzMax'
aliases = ['blitzmax', 'bmax']
filenames = ['*.bmx']
mimetypes = ['text/x-bmx']
bmax_vopwords = r'\b(Shl|Shr|Sar|Mod)\b'
bmax_sktypes = r'@{1,2}|[!#$%]'
bmax_lktypes = r'\b(Int|Byte|Short|Float|Double|Long)\b'
bmax_name = r'[a-z_]\w*'
bmax_var = (r'(%s)(?:(?:([ \t]*)(%s)|([ \t]*:[ \t]*\b(?:Shl|Shr|Sar|Mod)\b)'
r'|([ \t]*)(:)([ \t]*)(?:%s|(%s)))(?:([ \t]*)(Ptr))?)') % \
(bmax_name, bmax_sktypes, bmax_lktypes, bmax_name)
bmax_func = bmax_var + r'?((?:[ \t]|\.\.\n)*)([(])'
flags = re.MULTILINE | re.IGNORECASE
tokens = {
'root': [
# Text
(r'[ \t]+', Text),
(r'\.\.\n', Text), # Line continuation
# Comments
(r"'.*?\n", Comment.Single),
(r'([ \t]*)\bRem\n(\n|.)*?\s*\bEnd([ \t]*)Rem', Comment.Multiline),
# Data types
('"', String.Double, 'string'),
# Numbers
(r'[0-9]+\.[0-9]*(?!\.)', Number.Float),
(r'\.[0-9]*(?!\.)', Number.Float),
(r'[0-9]+', Number.Integer),
(r'\$[0-9a-f]+', Number.Hex),
(r'\%[10]+', Number.Bin),
# Other
(r'(?:(?:(:)?([ \t]*)(:?%s|([+\-*/&|~]))|Or|And|Not|[=<>^]))' %
(bmax_vopwords), Operator),
(r'[(),.:\[\]]', Punctuation),
(r'(?:#[\w \t]*)', Name.Label),
(r'(?:\?[\w \t]*)', Comment.Preproc),
# Identifiers
(r'\b(New)\b([ \t]?)([(]?)(%s)' % (bmax_name),
bygroups(Keyword.Reserved, Text, Punctuation, Name.Class)),
(r'\b(Import|Framework|Module)([ \t]+)(%s\.%s)' %
(bmax_name, bmax_name),
bygroups(Keyword.Reserved, Text, Keyword.Namespace)),
(bmax_func, bygroups(Name.Function, Text, Keyword.Type,
Operator, Text, Punctuation, Text,
Keyword.Type, Name.Class, Text,
Keyword.Type, Text, Punctuation)),
(bmax_var, bygroups(Name.Variable, Text, Keyword.Type, Operator,
Text, Punctuation, Text, Keyword.Type,
Name.Class, Text, Keyword.Type)),
(r'\b(Type|Extends)([ \t]+)(%s)' % (bmax_name),
bygroups(Keyword.Reserved, Text, Name.Class)),
# Keywords
(r'\b(Ptr)\b', Keyword.Type),
(r'\b(Pi|True|False|Null|Self|Super)\b', Keyword.Constant),
(r'\b(Local|Global|Const|Field)\b', Keyword.Declaration),
(words((
'TNullMethodException', 'TNullFunctionException',
'TNullObjectException', 'TArrayBoundsException',
'TRuntimeException'), prefix=r'\b', suffix=r'\b'), Name.Exception),
(words((
'Strict', 'SuperStrict', 'Module', 'ModuleInfo',
'End', 'Return', 'Continue', 'Exit', 'Public', 'Private',
'Var', 'VarPtr', 'Chr', 'Len', 'Asc', 'SizeOf', 'Sgn', 'Abs', 'Min', 'Max',
'New', 'Release', 'Delete', 'Incbin', 'IncbinPtr', 'IncbinLen',
'Framework', 'Include', 'Import', 'Extern', 'EndExtern',
'Function', 'EndFunction', 'Type', 'EndType', 'Extends', 'Method', 'EndMethod',
'Abstract', 'Final', 'If', 'Then', 'Else', 'ElseIf', 'EndIf',
'For', 'To', 'Next', 'Step', 'EachIn', 'While', 'Wend', 'EndWhile',
'Repeat', 'Until', 'Forever', 'Select', 'Case', 'Default', 'EndSelect',
'Try', 'Catch', 'EndTry', 'Throw', 'Assert', 'Goto', 'DefData', 'ReadData',
'RestoreData'), prefix=r'\b', suffix=r'\b'),
Keyword.Reserved),
# Final resolve (for variable names and such)
(r'(%s)' % (bmax_name), Name.Variable),
],
'string': [
(r'""', String.Double),
(r'"C?', String.Double, '#pop'),
(r'[^"]+', String.Double),
],
}
class BlitzBasicLexer(RegexLexer):
"""
For `BlitzBasic <http://blitzbasic.com>`_ source code.
.. versionadded:: 2.0
"""
name = 'BlitzBasic'
aliases = ['blitzbasic', 'b3d', 'bplus']
filenames = ['*.bb', '*.decls']
mimetypes = ['text/x-bb']
bb_sktypes = r'@{1,2}|[#$%]'
bb_name = r'[a-z]\w*'
bb_var = (r'(%s)(?:([ \t]*)(%s)|([ \t]*)([.])([ \t]*)(?:(%s)))?') % \
(bb_name, bb_sktypes, bb_name)
flags = re.MULTILINE | re.IGNORECASE
tokens = {
'root': [
# Text
(r'[ \t]+', Text),
# Comments
(r";.*?\n", Comment.Single),
# Data types
('"', String.Double, 'string'),
# Numbers
(r'[0-9]+\.[0-9]*(?!\.)', Number.Float),
(r'\.[0-9]+(?!\.)', Number.Float),
(r'[0-9]+', Number.Integer),
(r'\$[0-9a-f]+', Number.Hex),
(r'\%[10]+', Number.Bin),
# Other
(words(('Shl', 'Shr', 'Sar', 'Mod', 'Or', 'And', 'Not',
'Abs', 'Sgn', 'Handle', 'Int', 'Float', 'Str',
'First', 'Last', 'Before', 'After'),
prefix=r'\b', suffix=r'\b'),
Operator),
(r'([+\-*/~=<>^])', Operator),
(r'[(),:\[\]\\]', Punctuation),
(r'\.([ \t]*)(%s)' % bb_name, Name.Label),
# Identifiers
(r'\b(New)\b([ \t]+)(%s)' % (bb_name),
bygroups(Keyword.Reserved, Text, Name.Class)),
(r'\b(Gosub|Goto)\b([ \t]+)(%s)' % (bb_name),
bygroups(Keyword.Reserved, Text, Name.Label)),
(r'\b(Object)\b([ \t]*)([.])([ \t]*)(%s)\b' % (bb_name),
bygroups(Operator, Text, Punctuation, Text, Name.Class)),
(r'\b%s\b([ \t]*)(\()' % bb_var,
bygroups(Name.Function, Text, Keyword.Type, Text, Punctuation,
Text, Name.Class, Text, Punctuation)),
(r'\b(Function)\b([ \t]+)%s' % bb_var,
bygroups(Keyword.Reserved, Text, Name.Function, Text, Keyword.Type,
Text, Punctuation, Text, Name.Class)),
(r'\b(Type)([ \t]+)(%s)' % (bb_name),
bygroups(Keyword.Reserved, Text, Name.Class)),
# Keywords
(r'\b(Pi|True|False|Null)\b', Keyword.Constant),
(r'\b(Local|Global|Const|Field|Dim)\b', Keyword.Declaration),
(words((
'End', 'Return', 'Exit', 'Chr', 'Len', 'Asc', 'New', 'Delete', 'Insert',
'Include', 'Function', 'Type', 'If', 'Then', 'Else', 'ElseIf', 'EndIf',
'For', 'To', 'Next', 'Step', 'Each', 'While', 'Wend',
'Repeat', 'Until', 'Forever', 'Select', 'Case', 'Default',
'Goto', 'Gosub', 'Data', 'Read', 'Restore'), prefix=r'\b', suffix=r'\b'),
Keyword.Reserved),
# Final resolve (for variable names and such)
# (r'(%s)' % (bb_name), Name.Variable),
(bb_var, bygroups(Name.Variable, Text, Keyword.Type,
Text, Punctuation, Text, Name.Class)),
],
'string': [
(r'""', String.Double),
(r'"C?', String.Double, '#pop'),
(r'[^"]+', String.Double),
],
}
class MonkeyLexer(RegexLexer):
"""
For
`Monkey <https://en.wikipedia.org/wiki/Monkey_(programming_language)>`_
source code.
.. versionadded:: 1.6
"""
name = 'Monkey'
aliases = ['monkey']
filenames = ['*.monkey']
mimetypes = ['text/x-monkey']
name_variable = r'[a-z_]\w*'
name_function = r'[A-Z]\w*'
name_constant = r'[A-Z_][A-Z0-9_]*'
name_class = r'[A-Z]\w*'
name_module = r'[a-z0-9_]*'
keyword_type = r'(?:Int|Float|String|Bool|Object|Array|Void)'
# ? == Bool // % == Int // # == Float // $ == String
keyword_type_special = r'[?%#$]'
flags = re.MULTILINE
tokens = {
'root': [
# Text
(r'\s+', Text),
# Comments
(r"'.*", Comment),
(r'(?i)^#rem\b', Comment.Multiline, 'comment'),
# preprocessor directives
(r'(?i)^(?:#If|#ElseIf|#Else|#EndIf|#End|#Print|#Error)\b', Comment.Preproc),
# preprocessor variable (any line starting with '#' that is not a directive)
(r'^#', Comment.Preproc, 'variables'),
# String
('"', String.Double, 'string'),
# Numbers
(r'[0-9]+\.[0-9]*(?!\.)', Number.Float),
(r'\.[0-9]+(?!\.)', Number.Float),
(r'[0-9]+', Number.Integer),
(r'\$[0-9a-fA-Z]+', Number.Hex),
(r'\%[10]+', Number.Bin),
# Native data types
(r'\b%s\b' % keyword_type, Keyword.Type),
# Exception handling
(r'(?i)\b(?:Try|Catch|Throw)\b', Keyword.Reserved),
(r'Throwable', Name.Exception),
# Builtins
(r'(?i)\b(?:Null|True|False)\b', Name.Builtin),
(r'(?i)\b(?:Self|Super)\b', Name.Builtin.Pseudo),
(r'\b(?:HOST|LANG|TARGET|CONFIG)\b', Name.Constant),
# Keywords
(r'(?i)^(Import)(\s+)(.*)(\n)',
bygroups(Keyword.Namespace, Text, Name.Namespace, Text)),
(r'(?i)^Strict\b.*\n', Keyword.Reserved),
(r'(?i)(Const|Local|Global|Field)(\s+)',
bygroups(Keyword.Declaration, Text), 'variables'),
(r'(?i)(New|Class|Interface|Extends|Implements)(\s+)',
bygroups(Keyword.Reserved, Text), 'classname'),
(r'(?i)(Function|Method)(\s+)',
bygroups(Keyword.Reserved, Text), 'funcname'),
(r'(?i)(?:End|Return|Public|Private|Extern|Property|'
r'Final|Abstract)\b', Keyword.Reserved),
# Flow Control stuff
(r'(?i)(?:If|Then|Else|ElseIf|EndIf|'
r'Select|Case|Default|'
r'While|Wend|'
r'Repeat|Until|Forever|'
r'For|To|Until|Step|EachIn|Next|'
r'Exit|Continue)\s+', Keyword.Reserved),
# not used yet
(r'(?i)\b(?:Module|Inline)\b', Keyword.Reserved),
# Array
(r'[\[\]]', Punctuation),
# Other
(r'<=|>=|<>|\*=|/=|\+=|-=|&=|~=|\|=|[-&*/^+=<>|~]', Operator),
(r'(?i)(?:Not|Mod|Shl|Shr|And|Or)', Operator.Word),
(r'[(){}!#,.:]', Punctuation),
# catch the rest
(r'%s\b' % name_constant, Name.Constant),
(r'%s\b' % name_function, Name.Function),
(r'%s\b' % name_variable, Name.Variable),
],
'funcname': [
(r'(?i)%s\b' % name_function, Name.Function),
(r':', Punctuation, 'classname'),
(r'\s+', Text),
(r'\(', Punctuation, 'variables'),
(r'\)', Punctuation, '#pop')
],
'classname': [
(r'%s\.' % name_module, Name.Namespace),
(r'%s\b' % keyword_type, Keyword.Type),
(r'%s\b' % name_class, Name.Class),
# array (of given size)
(r'(\[)(\s*)(\d*)(\s*)(\])',
bygroups(Punctuation, Text, Number.Integer, Text, Punctuation)),
# generics
(r'\s+(?!<)', Text, '#pop'),
(r'<', Punctuation, '#push'),
(r'>', Punctuation, '#pop'),
(r'\n', Text, '#pop'),
default('#pop')
],
'variables': [
(r'%s\b' % name_constant, Name.Constant),
(r'%s\b' % name_variable, Name.Variable),
(r'%s' % keyword_type_special, Keyword.Type),
(r'\s+', Text),
(r':', Punctuation, 'classname'),
(r',', Punctuation, '#push'),
default('#pop')
],
'string': [
(r'[^"~]+', String.Double),
(r'~q|~n|~r|~t|~z|~~', String.Escape),
(r'"', String.Double, '#pop'),
],
'comment': [
(r'(?i)^#rem.*?', Comment.Multiline, "#push"),
(r'(?i)^#end.*?', Comment.Multiline, "#pop"),
(r'\n', Comment.Multiline),
(r'.+', Comment.Multiline),
],
}
class CbmBasicV2Lexer(RegexLexer):
"""
For CBM BASIC V2 sources.
.. versionadded:: 1.6
"""
name = 'CBM BASIC V2'
aliases = ['cbmbas']
filenames = ['*.bas']
flags = re.IGNORECASE
tokens = {
'root': [
(r'rem.*\n', Comment.Single),
(r'\s+', Text),
(r'new|run|end|for|to|next|step|go(to|sub)?|on|return|stop|cont'
r'|if|then|input#?|read|wait|load|save|verify|poke|sys|print#?'
r'|list|clr|cmd|open|close|get#?', Keyword.Reserved),
(r'data|restore|dim|let|def|fn', Keyword.Declaration),
(r'tab|spc|sgn|int|abs|usr|fre|pos|sqr|rnd|log|exp|cos|sin|tan|atn'
r'|peek|len|val|asc|(str|chr|left|right|mid)\$', Name.Builtin),
(r'[-+*/^<>=]', Operator),
(r'not|and|or', Operator.Word),
(r'"[^"\n]*.', String),
(r'\d+|[-+]?\d*\.\d*(e[-+]?\d+)?', Number.Float),
(r'[(),:;]', Punctuation),
(r'\w+[$%]?', Name),
]
}
def analyse_text(self, text):
# if it starts with a line number, it shouldn't be a "modern" Basic
# like VB.net
if re.match(r'\d+', text):
return 0.2
class QBasicLexer(RegexLexer):
"""
For
`QBasic <http://en.wikipedia.org/wiki/QBasic>`_
source code.
.. versionadded:: 2.0
"""
name = 'QBasic'
aliases = ['qbasic', 'basic']
filenames = ['*.BAS', '*.bas']
mimetypes = ['text/basic']
declarations = ('DATA', 'LET')
functions = (
'ABS', 'ASC', 'ATN', 'CDBL', 'CHR$', 'CINT', 'CLNG',
'COMMAND$', 'COS', 'CSNG', 'CSRLIN', 'CVD', 'CVDMBF', 'CVI',
'CVL', 'CVS', 'CVSMBF', 'DATE$', 'ENVIRON$', 'EOF', 'ERDEV',
'ERDEV$', 'ERL', 'ERR', 'EXP', 'FILEATTR', 'FIX', 'FRE',
'FREEFILE', 'HEX$', 'INKEY$', 'INP', 'INPUT$', 'INSTR', 'INT',
'IOCTL$', 'LBOUND', 'LCASE$', 'LEFT$', 'LEN', 'LOC', 'LOF',
'LOG', 'LPOS', 'LTRIM$', 'MID$', 'MKD$', 'MKDMBF$', 'MKI$',
'MKL$', 'MKS$', 'MKSMBF$', 'OCT$', 'PEEK', 'PEN', 'PLAY',
'PMAP', 'POINT', 'POS', 'RIGHT$', 'RND', 'RTRIM$', 'SADD',
'SCREEN', 'SEEK', 'SETMEM', 'SGN', 'SIN', 'SPACE$', 'SPC',
'SQR', 'STICK', 'STR$', 'STRIG', 'STRING$', 'TAB', 'TAN',
'TIME$', 'TIMER', 'UBOUND', 'UCASE$', 'VAL', 'VARPTR',
'VARPTR$', 'VARSEG'
)
metacommands = ('$DYNAMIC', '$INCLUDE', '$STATIC')
operators = ('AND', 'EQV', 'IMP', 'NOT', 'OR', 'XOR')
statements = (
'BEEP', 'BLOAD', 'BSAVE', 'CALL', 'CALL ABSOLUTE',
'CALL INTERRUPT', 'CALLS', 'CHAIN', 'CHDIR', 'CIRCLE', 'CLEAR',
'CLOSE', 'CLS', 'COLOR', 'COM', 'COMMON', 'CONST', 'DATA',
'DATE$', 'DECLARE', 'DEF FN', 'DEF SEG', 'DEFDBL', 'DEFINT',
'DEFLNG', 'DEFSNG', 'DEFSTR', 'DEF', 'DIM', 'DO', 'LOOP',
'DRAW', 'END', 'ENVIRON', 'ERASE', 'ERROR', 'EXIT', 'FIELD',
'FILES', 'FOR', 'NEXT', 'FUNCTION', 'GET', 'GOSUB', 'GOTO',
'IF', 'THEN', 'INPUT', 'INPUT #', 'IOCTL', 'KEY', 'KEY',
'KILL', 'LET', 'LINE', 'LINE INPUT', 'LINE INPUT #', 'LOCATE',
'LOCK', 'UNLOCK', 'LPRINT', 'LSET', 'MID$', 'MKDIR', 'NAME',
'ON COM', 'ON ERROR', 'ON KEY', 'ON PEN', 'ON PLAY',
'ON STRIG', 'ON TIMER', 'ON UEVENT', 'ON', 'OPEN', 'OPEN COM',
'OPTION BASE', 'OUT', 'PAINT', 'PALETTE', 'PCOPY', 'PEN',
'PLAY', 'POKE', 'PRESET', 'PRINT', 'PRINT #', 'PRINT USING',
'PSET', 'PUT', 'PUT', 'RANDOMIZE', 'READ', 'REDIM', 'REM',
'RESET', 'RESTORE', 'RESUME', 'RETURN', 'RMDIR', 'RSET', 'RUN',
'SCREEN', 'SEEK', 'SELECT CASE', 'SHARED', 'SHELL', 'SLEEP',
'SOUND', 'STATIC', 'STOP', 'STRIG', 'SUB', 'SWAP', 'SYSTEM',
'TIME$', 'TIMER', 'TROFF', 'TRON', 'TYPE', 'UEVENT', 'UNLOCK',
'VIEW', 'WAIT', 'WHILE', 'WEND', 'WIDTH', 'WINDOW', 'WRITE'
)
keywords = (
'ACCESS', 'ALIAS', 'ANY', 'APPEND', 'AS', 'BASE', 'BINARY',
'BYVAL', 'CASE', 'CDECL', 'DOUBLE', 'ELSE', 'ELSEIF', 'ENDIF',
'INTEGER', 'IS', 'LIST', 'LOCAL', 'LONG', 'LOOP', 'MOD',
'NEXT', 'OFF', 'ON', 'OUTPUT', 'RANDOM', 'SIGNAL', 'SINGLE',
'STEP', 'STRING', 'THEN', 'TO', 'UNTIL', 'USING', 'WEND'
)
tokens = {
'root': [
(r'\n+', Text),
(r'\s+', Text.Whitespace),
(r'^(\s*)(\d*)(\s*)(REM .*)$',
bygroups(Text.Whitespace, Name.Label, Text.Whitespace,
Comment.Single)),
(r'^(\s*)(\d+)(\s*)',
bygroups(Text.Whitespace, Name.Label, Text.Whitespace)),
(r'(?=[\s]*)(\w+)(?=[\s]*=)', Name.Variable.Global),
(r'(?=[^"]*)\'.*$', Comment.Single),
(r'"[^\n"]*"', String.Double),
(r'(END)(\s+)(FUNCTION|IF|SELECT|SUB)',
bygroups(Keyword.Reserved, Text.Whitespace, Keyword.Reserved)),
(r'(DECLARE)(\s+)([A-Z]+)(\s+)(\S+)',
bygroups(Keyword.Declaration, Text.Whitespace, Name.Variable,
Text.Whitespace, Name)),
(r'(DIM)(\s+)(SHARED)(\s+)([^\s(]+)',
bygroups(Keyword.Declaration, Text.Whitespace, Name.Variable,
Text.Whitespace, Name.Variable.Global)),
(r'(DIM)(\s+)([^\s(]+)',
bygroups(Keyword.Declaration, Text.Whitespace, Name.Variable.Global)),
(r'^(\s*)([a-zA-Z_]+)(\s*)(\=)',
bygroups(Text.Whitespace, Name.Variable.Global, Text.Whitespace,
Operator)),
(r'(GOTO|GOSUB)(\s+)(\w+\:?)',
bygroups(Keyword.Reserved, Text.Whitespace, Name.Label)),
(r'(SUB)(\s+)(\w+\:?)',
bygroups(Keyword.Reserved, Text.Whitespace, Name.Label)),
include('declarations'),
include('functions'),
include('metacommands'),
include('operators'),
include('statements'),
include('keywords'),
(r'[a-zA-Z_]\w*[$@#&!]', Name.Variable.Global),
(r'[a-zA-Z_]\w*\:', Name.Label),
(r'\-?\d*\.\d+[@|#]?', Number.Float),
(r'\-?\d+[@|#]', Number.Float),
(r'\-?\d+#?', Number.Integer.Long),
(r'\-?\d+#?', Number.Integer),
(r'!=|==|:=|\.=|<<|>>|[-~+/\\*%=<>&^|?:!.]', Operator),
(r'[\[\]{}(),;]', Punctuation),
(r'[\w]+', Name.Variable.Global),
],
# can't use regular \b because of X$()
# XXX: use words() here
'declarations': [
(r'\b(%s)(?=\(|\b)' % '|'.join(map(re.escape, declarations)),
Keyword.Declaration),
],
'functions': [
(r'\b(%s)(?=\(|\b)' % '|'.join(map(re.escape, functions)),
Keyword.Reserved),
],
'metacommands': [
(r'\b(%s)(?=\(|\b)' % '|'.join(map(re.escape, metacommands)),
Keyword.Constant),
],
'operators': [
(r'\b(%s)(?=\(|\b)' % '|'.join(map(re.escape, operators)), Operator.Word),
],
'statements': [
(r'\b(%s)\b' % '|'.join(map(re.escape, statements)),
Keyword.Reserved),
],
'keywords': [
(r'\b(%s)\b' % '|'.join(keywords), Keyword),
],
}
def analyse_text(text):
if '$DYNAMIC' in text or '$STATIC' in text:
return 0.9

View File

@ -1,160 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexers.bibtex
~~~~~~~~~~~~~~~~~~~~~~
Lexers for BibTeX bibliography data and styles
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import re
from pygments.lexer import RegexLexer, ExtendedRegexLexer, include, default, \
words
from pygments.token import Name, Comment, String, Error, Number, Text, \
Keyword, Punctuation
__all__ = ['BibTeXLexer', 'BSTLexer']
class BibTeXLexer(ExtendedRegexLexer):
"""
A lexer for BibTeX bibliography data format.
.. versionadded:: 2.2
"""
name = 'BibTeX'
aliases = ['bib', 'bibtex']
filenames = ['*.bib']
mimetypes = ["text/x-bibtex"]
flags = re.IGNORECASE
ALLOWED_CHARS = r'@!$&*+\-./:;<>?\[\\\]^`|~'
IDENTIFIER = '[{0}][{1}]*'.format('a-z_' + ALLOWED_CHARS, r'\w' + ALLOWED_CHARS)
def open_brace_callback(self, match, ctx):
opening_brace = match.group()
ctx.opening_brace = opening_brace
yield match.start(), Punctuation, opening_brace
ctx.pos = match.end()
def close_brace_callback(self, match, ctx):
closing_brace = match.group()
if (
ctx.opening_brace == '{' and closing_brace != '}' or
ctx.opening_brace == '(' and closing_brace != ')'
):
yield match.start(), Error, closing_brace
else:
yield match.start(), Punctuation, closing_brace
del ctx.opening_brace
ctx.pos = match.end()
tokens = {
'root': [
include('whitespace'),
('@comment', Comment),
('@preamble', Name.Class, ('closing-brace', 'value', 'opening-brace')),
('@string', Name.Class, ('closing-brace', 'field', 'opening-brace')),
('@' + IDENTIFIER, Name.Class,
('closing-brace', 'command-body', 'opening-brace')),
('.+', Comment),
],
'opening-brace': [
include('whitespace'),
(r'[{(]', open_brace_callback, '#pop'),
],
'closing-brace': [
include('whitespace'),
(r'[})]', close_brace_callback, '#pop'),
],
'command-body': [
include('whitespace'),
(r'[^\s\,\}]+', Name.Label, ('#pop', 'fields')),
],
'fields': [
include('whitespace'),
(',', Punctuation, 'field'),
default('#pop'),
],
'field': [
include('whitespace'),
(IDENTIFIER, Name.Attribute, ('value', '=')),
default('#pop'),
],
'=': [
include('whitespace'),
('=', Punctuation, '#pop'),
],
'value': [
include('whitespace'),
(IDENTIFIER, Name.Variable),
('"', String, 'quoted-string'),
(r'\{', String, 'braced-string'),
(r'[\d]+', Number),
('#', Punctuation),
default('#pop'),
],
'quoted-string': [
(r'\{', String, 'braced-string'),
('"', String, '#pop'),
('[^\{\"]+', String),
],
'braced-string': [
(r'\{', String, '#push'),
(r'\}', String, '#pop'),
('[^\{\}]+', String),
],
'whitespace': [
(r'\s+', Text),
],
}
class BSTLexer(RegexLexer):
"""
A lexer for BibTeX bibliography styles.
.. versionadded:: 2.2
"""
name = 'BST'
aliases = ['bst', 'bst-pybtex']
filenames = ['*.bst']
flags = re.IGNORECASE | re.MULTILINE
tokens = {
'root': [
include('whitespace'),
(words(['read', 'sort']), Keyword),
(words(['execute', 'integers', 'iterate', 'reverse', 'strings']),
Keyword, ('group')),
(words(['function', 'macro']), Keyword, ('group', 'group')),
(words(['entry']), Keyword, ('group', 'group', 'group')),
],
'group': [
include('whitespace'),
(r'\{', Punctuation, ('#pop', 'group-end', 'body')),
],
'group-end': [
include('whitespace'),
(r'\}', Punctuation, '#pop'),
],
'body': [
include('whitespace'),
(r"\'[^#\"\{\}\s]+", Name.Function),
(r'[^#\"\{\}\s]+\$', Name.Builtin),
(r'[^#\"\{\}\s]+', Name.Variable),
(r'"[^\"]*"', String),
(r'#-?\d+', Number),
(r'\{', Punctuation, ('group-end', 'body')),
default('#pop'),
],
'whitespace': [
('\s+', Text),
('%.*?$', Comment.SingleLine),
],
}

View File

@ -1,612 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexers.business
~~~~~~~~~~~~~~~~~~~~~~~~
Lexers for "business-oriented" languages.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import re
from pygments.lexer import RegexLexer, include, words, bygroups
from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
Number, Punctuation, Error
from pygments.lexers._openedge_builtins import OPENEDGEKEYWORDS
__all__ = ['CobolLexer', 'CobolFreeformatLexer', 'ABAPLexer', 'OpenEdgeLexer',
'GoodDataCLLexer', 'MaqlLexer']
class CobolLexer(RegexLexer):
"""
Lexer for OpenCOBOL code.
.. versionadded:: 1.6
"""
name = 'COBOL'
aliases = ['cobol']
filenames = ['*.cob', '*.COB', '*.cpy', '*.CPY']
mimetypes = ['text/x-cobol']
flags = re.IGNORECASE | re.MULTILINE
# Data Types: by PICTURE and USAGE
# Operators: **, *, +, -, /, <, >, <=, >=, =, <>
# Logical (?): NOT, AND, OR
# Reserved words:
# http://opencobol.add1tocobol.com/#reserved-words
# Intrinsics:
# http://opencobol.add1tocobol.com/#does-opencobol-implement-any-intrinsic-functions
tokens = {
'root': [
include('comment'),
include('strings'),
include('core'),
include('nums'),
(r'[a-z0-9]([\w\-]*[a-z0-9]+)?', Name.Variable),
# (r'[\s]+', Text),
(r'[ \t]+', Text),
],
'comment': [
(r'(^.{6}[*/].*\n|^.{6}|\*>.*\n)', Comment),
],
'core': [
# Figurative constants
(r'(^|(?<=[^\w\-]))(ALL\s+)?'
r'((ZEROES)|(HIGH-VALUE|LOW-VALUE|QUOTE|SPACE|ZERO)(S)?)'
r'\s*($|(?=[^\w\-]))',
Name.Constant),
# Reserved words STATEMENTS and other bolds
(words((
'ACCEPT', 'ADD', 'ALLOCATE', 'CALL', 'CANCEL', 'CLOSE', 'COMPUTE',
'CONFIGURATION', 'CONTINUE', 'DATA', 'DELETE', 'DISPLAY', 'DIVIDE',
'DIVISION', 'ELSE', 'END', 'END-ACCEPT',
'END-ADD', 'END-CALL', 'END-COMPUTE', 'END-DELETE', 'END-DISPLAY',
'END-DIVIDE', 'END-EVALUATE', 'END-IF', 'END-MULTIPLY', 'END-OF-PAGE',
'END-PERFORM', 'END-READ', 'END-RETURN', 'END-REWRITE', 'END-SEARCH',
'END-START', 'END-STRING', 'END-SUBTRACT', 'END-UNSTRING', 'END-WRITE',
'ENVIRONMENT', 'EVALUATE', 'EXIT', 'FD', 'FILE', 'FILE-CONTROL', 'FOREVER',
'FREE', 'GENERATE', 'GO', 'GOBACK', 'IDENTIFICATION', 'IF', 'INITIALIZE',
'INITIATE', 'INPUT-OUTPUT', 'INSPECT', 'INVOKE', 'I-O-CONTROL', 'LINKAGE',
'LOCAL-STORAGE', 'MERGE', 'MOVE', 'MULTIPLY', 'OPEN', 'PERFORM',
'PROCEDURE', 'PROGRAM-ID', 'RAISE', 'READ', 'RELEASE', 'RESUME',
'RETURN', 'REWRITE', 'SCREEN', 'SD', 'SEARCH', 'SECTION', 'SET',
'SORT', 'START', 'STOP', 'STRING', 'SUBTRACT', 'SUPPRESS',
'TERMINATE', 'THEN', 'UNLOCK', 'UNSTRING', 'USE', 'VALIDATE',
'WORKING-STORAGE', 'WRITE'), prefix=r'(^|(?<=[^\w\-]))',
suffix=r'\s*($|(?=[^\w\-]))'),
Keyword.Reserved),
# Reserved words
(words((
'ACCESS', 'ADDRESS', 'ADVANCING', 'AFTER', 'ALL',
'ALPHABET', 'ALPHABETIC', 'ALPHABETIC-LOWER', 'ALPHABETIC-UPPER',
'ALPHANUMERIC', 'ALPHANUMERIC-EDITED', 'ALSO', 'ALTER', 'ALTERNATE'
'ANY', 'ARE', 'AREA', 'AREAS', 'ARGUMENT-NUMBER', 'ARGUMENT-VALUE', 'AS',
'ASCENDING', 'ASSIGN', 'AT', 'AUTO', 'AUTO-SKIP', 'AUTOMATIC',
'AUTOTERMINATE', 'BACKGROUND-COLOR', 'BASED', 'BEEP', 'BEFORE', 'BELL',
'BLANK', 'BLINK', 'BLOCK', 'BOTTOM', 'BY', 'BYTE-LENGTH', 'CHAINING',
'CHARACTER', 'CHARACTERS', 'CLASS', 'CODE', 'CODE-SET', 'COL',
'COLLATING', 'COLS', 'COLUMN', 'COLUMNS', 'COMMA', 'COMMAND-LINE',
'COMMIT', 'COMMON', 'CONSTANT', 'CONTAINS', 'CONTENT', 'CONTROL',
'CONTROLS', 'CONVERTING', 'COPY', 'CORR', 'CORRESPONDING', 'COUNT', 'CRT',
'CURRENCY', 'CURSOR', 'CYCLE', 'DATE', 'DAY', 'DAY-OF-WEEK', 'DE',
'DEBUGGING', 'DECIMAL-POINT', 'DECLARATIVES', 'DEFAULT', 'DELIMITED',
'DELIMITER', 'DEPENDING', 'DESCENDING', 'DETAIL', 'DISK',
'DOWN', 'DUPLICATES', 'DYNAMIC', 'EBCDIC',
'ENTRY', 'ENVIRONMENT-NAME', 'ENVIRONMENT-VALUE', 'EOL', 'EOP',
'EOS', 'ERASE', 'ERROR', 'ESCAPE', 'EXCEPTION',
'EXCLUSIVE', 'EXTEND', 'EXTERNAL', 'FILE-ID', 'FILLER', 'FINAL',
'FIRST', 'FIXED', 'FLOAT-LONG', 'FLOAT-SHORT',
'FOOTING', 'FOR', 'FOREGROUND-COLOR', 'FORMAT', 'FROM', 'FULL',
'FUNCTION', 'FUNCTION-ID', 'GIVING', 'GLOBAL', 'GROUP',
'HEADING', 'HIGHLIGHT', 'I-O', 'ID',
'IGNORE', 'IGNORING', 'IN', 'INDEX', 'INDEXED', 'INDICATE',
'INITIAL', 'INITIALIZED', 'INPUT', 'INTO', 'INTRINSIC', 'INVALID',
'IS', 'JUST', 'JUSTIFIED', 'KEY', 'LABEL',
'LAST', 'LEADING', 'LEFT', 'LENGTH', 'LIMIT', 'LIMITS', 'LINAGE',
'LINAGE-COUNTER', 'LINE', 'LINES', 'LOCALE', 'LOCK',
'LOWLIGHT', 'MANUAL', 'MEMORY', 'MINUS', 'MODE', 'MULTIPLE',
'NATIONAL', 'NATIONAL-EDITED', 'NATIVE', 'NEGATIVE', 'NEXT', 'NO',
'NULL', 'NULLS', 'NUMBER', 'NUMBERS', 'NUMERIC', 'NUMERIC-EDITED',
'OBJECT-COMPUTER', 'OCCURS', 'OF', 'OFF', 'OMITTED', 'ON', 'ONLY',
'OPTIONAL', 'ORDER', 'ORGANIZATION', 'OTHER', 'OUTPUT', 'OVERFLOW',
'OVERLINE', 'PACKED-DECIMAL', 'PADDING', 'PAGE', 'PARAGRAPH',
'PLUS', 'POINTER', 'POSITION', 'POSITIVE', 'PRESENT', 'PREVIOUS',
'PRINTER', 'PRINTING', 'PROCEDURE-POINTER', 'PROCEDURES',
'PROCEED', 'PROGRAM', 'PROGRAM-POINTER', 'PROMPT', 'QUOTE',
'QUOTES', 'RANDOM', 'RD', 'RECORD', 'RECORDING', 'RECORDS', 'RECURSIVE',
'REDEFINES', 'REEL', 'REFERENCE', 'RELATIVE', 'REMAINDER', 'REMOVAL',
'RENAMES', 'REPLACING', 'REPORT', 'REPORTING', 'REPORTS', 'REPOSITORY',
'REQUIRED', 'RESERVE', 'RETURNING', 'REVERSE-VIDEO', 'REWIND',
'RIGHT', 'ROLLBACK', 'ROUNDED', 'RUN', 'SAME', 'SCROLL',
'SECURE', 'SEGMENT-LIMIT', 'SELECT', 'SENTENCE', 'SEPARATE',
'SEQUENCE', 'SEQUENTIAL', 'SHARING', 'SIGN', 'SIGNED', 'SIGNED-INT',
'SIGNED-LONG', 'SIGNED-SHORT', 'SIZE', 'SORT-MERGE', 'SOURCE',
'SOURCE-COMPUTER', 'SPECIAL-NAMES', 'STANDARD',
'STANDARD-1', 'STANDARD-2', 'STATUS', 'SUM',
'SYMBOLIC', 'SYNC', 'SYNCHRONIZED', 'TALLYING', 'TAPE',
'TEST', 'THROUGH', 'THRU', 'TIME', 'TIMES', 'TO', 'TOP', 'TRAILING',
'TRANSFORM', 'TYPE', 'UNDERLINE', 'UNIT', 'UNSIGNED',
'UNSIGNED-INT', 'UNSIGNED-LONG', 'UNSIGNED-SHORT', 'UNTIL', 'UP',
'UPDATE', 'UPON', 'USAGE', 'USING', 'VALUE', 'VALUES', 'VARYING',
'WAIT', 'WHEN', 'WITH', 'WORDS', 'YYYYDDD', 'YYYYMMDD'),
prefix=r'(^|(?<=[^\w\-]))', suffix=r'\s*($|(?=[^\w\-]))'),
Keyword.Pseudo),
# inactive reserved words
(words((
'ACTIVE-CLASS', 'ALIGNED', 'ANYCASE', 'ARITHMETIC', 'ATTRIBUTE',
'B-AND', 'B-NOT', 'B-OR', 'B-XOR', 'BIT', 'BOOLEAN', 'CD', 'CENTER',
'CF', 'CH', 'CHAIN', 'CLASS-ID', 'CLASSIFICATION', 'COMMUNICATION',
'CONDITION', 'DATA-POINTER', 'DESTINATION', 'DISABLE', 'EC', 'EGI',
'EMI', 'ENABLE', 'END-RECEIVE', 'ENTRY-CONVENTION', 'EO', 'ESI',
'EXCEPTION-OBJECT', 'EXPANDS', 'FACTORY', 'FLOAT-BINARY-16',
'FLOAT-BINARY-34', 'FLOAT-BINARY-7', 'FLOAT-DECIMAL-16',
'FLOAT-DECIMAL-34', 'FLOAT-EXTENDED', 'FORMAT', 'FUNCTION-POINTER',
'GET', 'GROUP-USAGE', 'IMPLEMENTS', 'INFINITY', 'INHERITS',
'INTERFACE', 'INTERFACE-ID', 'INVOKE', 'LC_ALL', 'LC_COLLATE',
'LC_CTYPE', 'LC_MESSAGES', 'LC_MONETARY', 'LC_NUMERIC', 'LC_TIME',
'LINE-COUNTER', 'MESSAGE', 'METHOD', 'METHOD-ID', 'NESTED', 'NONE',
'NORMAL', 'OBJECT', 'OBJECT-REFERENCE', 'OPTIONS', 'OVERRIDE',
'PAGE-COUNTER', 'PF', 'PH', 'PROPERTY', 'PROTOTYPE', 'PURGE',
'QUEUE', 'RAISE', 'RAISING', 'RECEIVE', 'RELATION', 'REPLACE',
'REPRESENTS-NOT-A-NUMBER', 'RESET', 'RESUME', 'RETRY', 'RF', 'RH',
'SECONDS', 'SEGMENT', 'SELF', 'SEND', 'SOURCES', 'STATEMENT',
'STEP', 'STRONG', 'SUB-QUEUE-1', 'SUB-QUEUE-2', 'SUB-QUEUE-3',
'SUPER', 'SYMBOL', 'SYSTEM-DEFAULT', 'TABLE', 'TERMINAL', 'TEXT',
'TYPEDEF', 'UCS-4', 'UNIVERSAL', 'USER-DEFAULT', 'UTF-16', 'UTF-8',
'VAL-STATUS', 'VALID', 'VALIDATE', 'VALIDATE-STATUS'),
prefix=r'(^|(?<=[^\w\-]))', suffix=r'\s*($|(?=[^\w\-]))'),
Error),
# Data Types
(r'(^|(?<=[^\w\-]))'
r'(PIC\s+.+?(?=(\s|\.\s))|PICTURE\s+.+?(?=(\s|\.\s))|'
r'(COMPUTATIONAL)(-[1-5X])?|(COMP)(-[1-5X])?|'
r'BINARY-C-LONG|'
r'BINARY-CHAR|BINARY-DOUBLE|BINARY-LONG|BINARY-SHORT|'
r'BINARY)\s*($|(?=[^\w\-]))', Keyword.Type),
# Operators
(r'(\*\*|\*|\+|-|/|<=|>=|<|>|==|/=|=)', Operator),
# (r'(::)', Keyword.Declaration),
(r'([(),;:&%.])', Punctuation),
# Intrinsics
(r'(^|(?<=[^\w\-]))(ABS|ACOS|ANNUITY|ASIN|ATAN|BYTE-LENGTH|'
r'CHAR|COMBINED-DATETIME|CONCATENATE|COS|CURRENT-DATE|'
r'DATE-OF-INTEGER|DATE-TO-YYYYMMDD|DAY-OF-INTEGER|DAY-TO-YYYYDDD|'
r'EXCEPTION-(?:FILE|LOCATION|STATEMENT|STATUS)|EXP10|EXP|E|'
r'FACTORIAL|FRACTION-PART|INTEGER-OF-(?:DATE|DAY|PART)|INTEGER|'
r'LENGTH|LOCALE-(?:DATE|TIME(?:-FROM-SECONDS)?)|LOG(?:10)?|'
r'LOWER-CASE|MAX|MEAN|MEDIAN|MIDRANGE|MIN|MOD|NUMVAL(?:-C)?|'
r'ORD(?:-MAX|-MIN)?|PI|PRESENT-VALUE|RANDOM|RANGE|REM|REVERSE|'
r'SECONDS-FROM-FORMATTED-TIME|SECONDS-PAST-MIDNIGHT|SIGN|SIN|SQRT|'
r'STANDARD-DEVIATION|STORED-CHAR-LENGTH|SUBSTITUTE(?:-CASE)?|'
r'SUM|TAN|TEST-DATE-YYYYMMDD|TEST-DAY-YYYYDDD|TRIM|'
r'UPPER-CASE|VARIANCE|WHEN-COMPILED|YEAR-TO-YYYY)\s*'
r'($|(?=[^\w\-]))', Name.Function),
# Booleans
(r'(^|(?<=[^\w\-]))(true|false)\s*($|(?=[^\w\-]))', Name.Builtin),
# Comparing Operators
(r'(^|(?<=[^\w\-]))(equal|equals|ne|lt|le|gt|ge|'
r'greater|less|than|not|and|or)\s*($|(?=[^\w\-]))', Operator.Word),
],
# \"[^\"\n]*\"|\'[^\'\n]*\'
'strings': [
# apparently strings can be delimited by EOL if they are continued
# in the next line
(r'"[^"\n]*("|\n)', String.Double),
(r"'[^'\n]*('|\n)", String.Single),
],
'nums': [
(r'\d+(\s*|\.$|$)', Number.Integer),
(r'[+-]?\d*\.\d+(E[-+]?\d+)?', Number.Float),
(r'[+-]?\d+\.\d*(E[-+]?\d+)?', Number.Float),
],
}
class CobolFreeformatLexer(CobolLexer):
"""
Lexer for Free format OpenCOBOL code.
.. versionadded:: 1.6
"""
name = 'COBOLFree'
aliases = ['cobolfree']
filenames = ['*.cbl', '*.CBL']
mimetypes = []
flags = re.IGNORECASE | re.MULTILINE
tokens = {
'comment': [
(r'(\*>.*\n|^\w*\*.*$)', Comment),
],
}
class ABAPLexer(RegexLexer):
"""
Lexer for ABAP, SAP's integrated language.
.. versionadded:: 1.1
"""
name = 'ABAP'
aliases = ['abap']
filenames = ['*.abap', '*.ABAP']
mimetypes = ['text/x-abap']
flags = re.IGNORECASE | re.MULTILINE
tokens = {
'common': [
(r'\s+', Text),
(r'^\*.*$', Comment.Single),
(r'\".*?\n', Comment.Single),
(r'##\w+', Comment.Special),
],
'variable-names': [
(r'<\S+>', Name.Variable),
(r'\w[\w~]*(?:(\[\])|->\*)?', Name.Variable),
],
'root': [
include('common'),
# function calls
(r'CALL\s+(?:BADI|CUSTOMER-FUNCTION|FUNCTION)',
Keyword),
(r'(CALL\s+(?:DIALOG|SCREEN|SUBSCREEN|SELECTION-SCREEN|'
r'TRANSACTION|TRANSFORMATION))\b',
Keyword),
(r'(FORM|PERFORM)(\s+)(\w+)',
bygroups(Keyword, Text, Name.Function)),
(r'(PERFORM)(\s+)(\()(\w+)(\))',
bygroups(Keyword, Text, Punctuation, Name.Variable, Punctuation)),
(r'(MODULE)(\s+)(\S+)(\s+)(INPUT|OUTPUT)',
bygroups(Keyword, Text, Name.Function, Text, Keyword)),
# method implementation
(r'(METHOD)(\s+)([\w~]+)',
bygroups(Keyword, Text, Name.Function)),
# method calls
(r'(\s+)([\w\-]+)([=\-]>)([\w\-~]+)',
bygroups(Text, Name.Variable, Operator, Name.Function)),
# call methodnames returning style
(r'(?<=(=|-)>)([\w\-~]+)(?=\()', Name.Function),
# text elements
(r'(TEXT)(-)(\d{3})',
bygroups(Keyword, Punctuation, Number.Integer)),
(r'(TEXT)(-)(\w{3})',
bygroups(Keyword, Punctuation, Name.Variable)),
# keywords with dashes in them.
# these need to be first, because for instance the -ID part
# of MESSAGE-ID wouldn't get highlighted if MESSAGE was
# first in the list of keywords.
(r'(ADD-CORRESPONDING|AUTHORITY-CHECK|'
r'CLASS-DATA|CLASS-EVENTS|CLASS-METHODS|CLASS-POOL|'
r'DELETE-ADJACENT|DIVIDE-CORRESPONDING|'
r'EDITOR-CALL|ENHANCEMENT-POINT|ENHANCEMENT-SECTION|EXIT-COMMAND|'
r'FIELD-GROUPS|FIELD-SYMBOLS|FUNCTION-POOL|'
r'INTERFACE-POOL|INVERTED-DATE|'
r'LOAD-OF-PROGRAM|LOG-POINT|'
r'MESSAGE-ID|MOVE-CORRESPONDING|MULTIPLY-CORRESPONDING|'
r'NEW-LINE|NEW-PAGE|NEW-SECTION|NO-EXTENSION|'
r'OUTPUT-LENGTH|PRINT-CONTROL|'
r'SELECT-OPTIONS|START-OF-SELECTION|SUBTRACT-CORRESPONDING|'
r'SYNTAX-CHECK|SYSTEM-EXCEPTIONS|'
r'TYPE-POOL|TYPE-POOLS|NO-DISPLAY'
r')\b', Keyword),
# keyword kombinations
(r'(?<![-\>])(CREATE\s+(PUBLIC|PRIVATE|DATA|OBJECT)|'
r'(PUBLIC|PRIVATE|PROTECTED)\s+SECTION|'
r'(TYPE|LIKE)\s+((LINE\s+OF|REF\s+TO|'
r'(SORTED|STANDARD|HASHED)\s+TABLE\s+OF))?|'
r'FROM\s+(DATABASE|MEMORY)|CALL\s+METHOD|'
r'(GROUP|ORDER) BY|HAVING|SEPARATED BY|'
r'GET\s+(BADI|BIT|CURSOR|DATASET|LOCALE|PARAMETER|'
r'PF-STATUS|(PROPERTY|REFERENCE)\s+OF|'
r'RUN\s+TIME|TIME\s+(STAMP)?)?|'
r'SET\s+(BIT|BLANK\s+LINES|COUNTRY|CURSOR|DATASET|EXTENDED\s+CHECK|'
r'HANDLER|HOLD\s+DATA|LANGUAGE|LEFT\s+SCROLL-BOUNDARY|'
r'LOCALE|MARGIN|PARAMETER|PF-STATUS|PROPERTY\s+OF|'
r'RUN\s+TIME\s+(ANALYZER|CLOCK\s+RESOLUTION)|SCREEN|'
r'TITLEBAR|UPADTE\s+TASK\s+LOCAL|USER-COMMAND)|'
r'CONVERT\s+((INVERTED-)?DATE|TIME|TIME\s+STAMP|TEXT)|'
r'(CLOSE|OPEN)\s+(DATASET|CURSOR)|'
r'(TO|FROM)\s+(DATA BUFFER|INTERNAL TABLE|MEMORY ID|'
r'DATABASE|SHARED\s+(MEMORY|BUFFER))|'
r'DESCRIBE\s+(DISTANCE\s+BETWEEN|FIELD|LIST|TABLE)|'
r'FREE\s(MEMORY|OBJECT)?|'
r'PROCESS\s+(BEFORE\s+OUTPUT|AFTER\s+INPUT|'
r'ON\s+(VALUE-REQUEST|HELP-REQUEST))|'
r'AT\s+(LINE-SELECTION|USER-COMMAND|END\s+OF|NEW)|'
r'AT\s+SELECTION-SCREEN(\s+(ON(\s+(BLOCK|(HELP|VALUE)-REQUEST\s+FOR|'
r'END\s+OF|RADIOBUTTON\s+GROUP))?|OUTPUT))?|'
r'SELECTION-SCREEN:?\s+((BEGIN|END)\s+OF\s+((TABBED\s+)?BLOCK|LINE|'
r'SCREEN)|COMMENT|FUNCTION\s+KEY|'
r'INCLUDE\s+BLOCKS|POSITION|PUSHBUTTON|'
r'SKIP|ULINE)|'
r'LEAVE\s+(LIST-PROCESSING|PROGRAM|SCREEN|'
r'TO LIST-PROCESSING|TO TRANSACTION)'
r'(ENDING|STARTING)\s+AT|'
r'FORMAT\s+(COLOR|INTENSIFIED|INVERSE|HOTSPOT|INPUT|FRAMES|RESET)|'
r'AS\s+(CHECKBOX|SUBSCREEN|WINDOW)|'
r'WITH\s+(((NON-)?UNIQUE)?\s+KEY|FRAME)|'
r'(BEGIN|END)\s+OF|'
r'DELETE(\s+ADJACENT\s+DUPLICATES\sFROM)?|'
r'COMPARING(\s+ALL\s+FIELDS)?|'
r'(INSERT|APPEND)(\s+INITIAL\s+LINE\s+(IN)?TO|\s+LINES\s+OF)?|'
r'IN\s+((BYTE|CHARACTER)\s+MODE|PROGRAM)|'
r'END-OF-(DEFINITION|PAGE|SELECTION)|'
r'WITH\s+FRAME(\s+TITLE)|'
r'(REPLACE|FIND)\s+((FIRST|ALL)\s+OCCURRENCES?\s+OF\s+)?(SUBSTRING|REGEX)?|'
r'MATCH\s+(LENGTH|COUNT|LINE|OFFSET)|'
r'(RESPECTING|IGNORING)\s+CASE|'
r'IN\s+UPDATE\s+TASK|'
r'(SOURCE|RESULT)\s+(XML)?|'
r'REFERENCE\s+INTO|'
# simple kombinations
r'AND\s+(MARK|RETURN)|CLIENT\s+SPECIFIED|CORRESPONDING\s+FIELDS\s+OF|'
r'IF\s+FOUND|FOR\s+EVENT|INHERITING\s+FROM|LEAVE\s+TO\s+SCREEN|'
r'LOOP\s+AT\s+(SCREEN)?|LOWER\s+CASE|MATCHCODE\s+OBJECT|MODIF\s+ID|'
r'MODIFY\s+SCREEN|NESTING\s+LEVEL|NO\s+INTERVALS|OF\s+STRUCTURE|'
r'RADIOBUTTON\s+GROUP|RANGE\s+OF|REF\s+TO|SUPPRESS DIALOG|'
r'TABLE\s+OF|UPPER\s+CASE|TRANSPORTING\s+NO\s+FIELDS|'
r'VALUE\s+CHECK|VISIBLE\s+LENGTH|HEADER\s+LINE|COMMON\s+PART)\b', Keyword),
# single word keywords.
(r'(^|(?<=(\s|\.)))(ABBREVIATED|ABSTRACT|ADD|ALIASES|ALIGN|ALPHA|'
r'ASSERT|AS|ASSIGN(ING)?|AT(\s+FIRST)?|'
r'BACK|BLOCK|BREAK-POINT|'
r'CASE|CATCH|CHANGING|CHECK|CLASS|CLEAR|COLLECT|COLOR|COMMIT|'
r'CREATE|COMMUNICATION|COMPONENTS?|COMPUTE|CONCATENATE|CONDENSE|'
r'CONSTANTS|CONTEXTS|CONTINUE|CONTROLS|COUNTRY|CURRENCY|'
r'DATA|DATE|DECIMALS|DEFAULT|DEFINE|DEFINITION|DEFERRED|DEMAND|'
r'DETAIL|DIRECTORY|DIVIDE|DO|DUMMY|'
r'ELSE(IF)?|ENDAT|ENDCASE|ENDCATCH|ENDCLASS|ENDDO|ENDFORM|ENDFUNCTION|'
r'ENDIF|ENDINTERFACE|ENDLOOP|ENDMETHOD|ENDMODULE|ENDSELECT|ENDTRY|ENDWHILE|'
r'ENHANCEMENT|EVENTS|EXACT|EXCEPTIONS?|EXIT|EXPONENT|EXPORT|EXPORTING|EXTRACT|'
r'FETCH|FIELDS?|FOR|FORM|FORMAT|FREE|FROM|FUNCTION|'
r'HIDE|'
r'ID|IF|IMPORT|IMPLEMENTATION|IMPORTING|IN|INCLUDE|INCLUDING|'
r'INDEX|INFOTYPES|INITIALIZATION|INTERFACE|INTERFACES|INTO|'
r'LANGUAGE|LEAVE|LENGTH|LINES|LOAD|LOCAL|'
r'JOIN|'
r'KEY|'
r'NEXT|'
r'MAXIMUM|MESSAGE|METHOD[S]?|MINIMUM|MODULE|MODIFIER|MODIFY|MOVE|MULTIPLY|'
r'NODES|NUMBER|'
r'OBLIGATORY|OBJECT|OF|OFF|ON|OTHERS|OVERLAY|'
r'PACK|PAD|PARAMETERS|PERCENTAGE|POSITION|PROGRAM|PROVIDE|PUBLIC|PUT|PF\d\d|'
r'RAISE|RAISING|RANGES?|READ|RECEIVE|REDEFINITION|REFRESH|REJECT|REPORT|RESERVE|'
r'RESUME|RETRY|RETURN|RETURNING|RIGHT|ROLLBACK|REPLACE|'
r'SCROLL|SEARCH|SELECT|SHIFT|SIGN|SINGLE|SIZE|SKIP|SORT|SPLIT|STATICS|STOP|'
r'STYLE|SUBMATCHES|SUBMIT|SUBTRACT|SUM(?!\()|SUMMARY|SUMMING|SUPPLY|'
r'TABLE|TABLES|TIMESTAMP|TIMES?|TIMEZONE|TITLE|\??TO|'
r'TOP-OF-PAGE|TRANSFER|TRANSLATE|TRY|TYPES|'
r'ULINE|UNDER|UNPACK|UPDATE|USING|'
r'VALUE|VALUES|VIA|VARYING|VARY|'
r'WAIT|WHEN|WHERE|WIDTH|WHILE|WITH|WINDOW|WRITE|XSD|ZERO)\b', Keyword),
# builtins
(r'(abs|acos|asin|atan|'
r'boolc|boolx|bit_set|'
r'char_off|charlen|ceil|cmax|cmin|condense|contains|'
r'contains_any_of|contains_any_not_of|concat_lines_of|cos|cosh|'
r'count|count_any_of|count_any_not_of|'
r'dbmaxlen|distance|'
r'escape|exp|'
r'find|find_end|find_any_of|find_any_not_of|floor|frac|from_mixed|'
r'insert|'
r'lines|log|log10|'
r'match|matches|'
r'nmax|nmin|numofchar|'
r'repeat|replace|rescale|reverse|round|'
r'segment|shift_left|shift_right|sign|sin|sinh|sqrt|strlen|'
r'substring|substring_after|substring_from|substring_before|substring_to|'
r'tan|tanh|to_upper|to_lower|to_mixed|translate|trunc|'
r'xstrlen)(\()\b', bygroups(Name.Builtin, Punctuation)),
(r'&[0-9]', Name),
(r'[0-9]+', Number.Integer),
# operators which look like variable names before
# parsing variable names.
(r'(?<=(\s|.))(AND|OR|EQ|NE|GT|LT|GE|LE|CO|CN|CA|NA|CS|NOT|NS|CP|NP|'
r'BYTE-CO|BYTE-CN|BYTE-CA|BYTE-NA|BYTE-CS|BYTE-NS|'
r'IS\s+(NOT\s+)?(INITIAL|ASSIGNED|REQUESTED|BOUND))\b', Operator.Word),
include('variable-names'),
# standard operators after variable names,
# because < and > are part of field symbols.
(r'[?*<>=\-+&]', Operator),
(r"'(''|[^'])*'", String.Single),
(r"`([^`])*`", String.Single),
(r"([|}])([^{}|]*?)([|{])",
bygroups(Punctuation, String.Single, Punctuation)),
(r'[/;:()\[\],.]', Punctuation),
(r'(!)(\w+)', bygroups(Operator, Name)),
],
}
class OpenEdgeLexer(RegexLexer):
"""
Lexer for `OpenEdge ABL (formerly Progress)
<http://web.progress.com/en/openedge/abl.html>`_ source code.
.. versionadded:: 1.5
"""
name = 'OpenEdge ABL'
aliases = ['openedge', 'abl', 'progress']
filenames = ['*.p', '*.cls']
mimetypes = ['text/x-openedge', 'application/x-openedge']
types = (r'(?i)(^|(?<=[^\w\-]))(CHARACTER|CHAR|CHARA|CHARAC|CHARACT|CHARACTE|'
r'COM-HANDLE|DATE|DATETIME|DATETIME-TZ|'
r'DECIMAL|DEC|DECI|DECIM|DECIMA|HANDLE|'
r'INT64|INTEGER|INT|INTE|INTEG|INTEGE|'
r'LOGICAL|LONGCHAR|MEMPTR|RAW|RECID|ROWID)\s*($|(?=[^\w\-]))')
keywords = words(OPENEDGEKEYWORDS,
prefix=r'(?i)(^|(?<=[^\w\-]))',
suffix=r'\s*($|(?=[^\w\-]))')
tokens = {
'root': [
(r'/\*', Comment.Multiline, 'comment'),
(r'\{', Comment.Preproc, 'preprocessor'),
(r'\s*&.*', Comment.Preproc),
(r'0[xX][0-9a-fA-F]+[LlUu]*', Number.Hex),
(r'(?i)(DEFINE|DEF|DEFI|DEFIN)\b', Keyword.Declaration),
(types, Keyword.Type),
(keywords, Name.Builtin),
(r'"(\\\\|\\"|[^"])*"', String.Double),
(r"'(\\\\|\\'|[^'])*'", String.Single),
(r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
(r'[0-9]+', Number.Integer),
(r'\s+', Text),
(r'[+*/=-]', Operator),
(r'[.:()]', Punctuation),
(r'.', Name.Variable), # Lazy catch-all
],
'comment': [
(r'[^*/]', Comment.Multiline),
(r'/\*', Comment.Multiline, '#push'),
(r'\*/', Comment.Multiline, '#pop'),
(r'[*/]', Comment.Multiline)
],
'preprocessor': [
(r'[^{}]', Comment.Preproc),
(r'\{', Comment.Preproc, '#push'),
(r'\}', Comment.Preproc, '#pop'),
],
}
class GoodDataCLLexer(RegexLexer):
"""
Lexer for `GoodData-CL
<http://github.com/gooddata/GoodData-CL/raw/master/cli/src/main/resources/\
com/gooddata/processor/COMMANDS.txt>`_
script files.
.. versionadded:: 1.4
"""
name = 'GoodData-CL'
aliases = ['gooddata-cl']
filenames = ['*.gdc']
mimetypes = ['text/x-gooddata-cl']
flags = re.IGNORECASE
tokens = {
'root': [
# Comments
(r'#.*', Comment.Single),
# Function call
(r'[a-z]\w*', Name.Function),
# Argument list
(r'\(', Punctuation, 'args-list'),
# Punctuation
(r';', Punctuation),
# Space is not significant
(r'\s+', Text)
],
'args-list': [
(r'\)', Punctuation, '#pop'),
(r',', Punctuation),
(r'[a-z]\w*', Name.Variable),
(r'=', Operator),
(r'"', String, 'string-literal'),
(r'[0-9]+(?:\.[0-9]+)?(?:e[+-]?[0-9]{1,3})?', Number),
# Space is not significant
(r'\s', Text)
],
'string-literal': [
(r'\\[tnrfbae"\\]', String.Escape),
(r'"', String, '#pop'),
(r'[^\\"]+', String)
]
}
class MaqlLexer(RegexLexer):
"""
Lexer for `GoodData MAQL
<https://secure.gooddata.com/docs/html/advanced.metric.tutorial.html>`_
scripts.
.. versionadded:: 1.4
"""
name = 'MAQL'
aliases = ['maql']
filenames = ['*.maql']
mimetypes = ['text/x-gooddata-maql', 'application/x-gooddata-maql']
flags = re.IGNORECASE
tokens = {
'root': [
# IDENTITY
(r'IDENTIFIER\b', Name.Builtin),
# IDENTIFIER
(r'\{[^}]+\}', Name.Variable),
# NUMBER
(r'[0-9]+(?:\.[0-9]+)?(?:e[+-]?[0-9]{1,3})?', Number),
# STRING
(r'"', String, 'string-literal'),
# RELATION
(r'\<\>|\!\=', Operator),
(r'\=|\>\=|\>|\<\=|\<', Operator),
# :=
(r'\:\=', Operator),
# OBJECT
(r'\[[^]]+\]', Name.Variable.Class),
# keywords
(words((
'DIMENSION', 'DIMENSIONS', 'BOTTOM', 'METRIC', 'COUNT', 'OTHER',
'FACT', 'WITH', 'TOP', 'OR', 'ATTRIBUTE', 'CREATE', 'PARENT',
'FALSE', 'ROW', 'ROWS', 'FROM', 'ALL', 'AS', 'PF', 'COLUMN',
'COLUMNS', 'DEFINE', 'REPORT', 'LIMIT', 'TABLE', 'LIKE', 'AND',
'BY', 'BETWEEN', 'EXCEPT', 'SELECT', 'MATCH', 'WHERE', 'TRUE',
'FOR', 'IN', 'WITHOUT', 'FILTER', 'ALIAS', 'WHEN', 'NOT', 'ON',
'KEYS', 'KEY', 'FULLSET', 'PRIMARY', 'LABELS', 'LABEL',
'VISUAL', 'TITLE', 'DESCRIPTION', 'FOLDER', 'ALTER', 'DROP',
'ADD', 'DATASET', 'DATATYPE', 'INT', 'BIGINT', 'DOUBLE', 'DATE',
'VARCHAR', 'DECIMAL', 'SYNCHRONIZE', 'TYPE', 'DEFAULT', 'ORDER',
'ASC', 'DESC', 'HYPERLINK', 'INCLUDE', 'TEMPLATE', 'MODIFY'),
suffix=r'\b'),
Keyword),
# FUNCNAME
(r'[a-z]\w*\b', Name.Function),
# Comments
(r'#.*', Comment.Single),
# Punctuation
(r'[,;()]', Punctuation),
# Space is not significant
(r'\s+', Text)
],
'string-literal': [
(r'\\[tnrfbae"\\]', String.Escape),
(r'"', String, '#pop'),
(r'[^\\"]+', String)
],
}

View File

@ -1,252 +0,0 @@
# -*- coding: utf-8 -*-
"""
pygments.lexers.c_cpp
~~~~~~~~~~~~~~~~~~~~~
Lexers for C/C++ languages.
:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import re
from pygments.lexer import RegexLexer, include, bygroups, using, \
this, inherit, default, words
from pygments.util import get_bool_opt
from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
Number, Punctuation, Error
__all__ = ['CLexer', 'CppLexer']
class CFamilyLexer(RegexLexer):
"""
For C family source code. This is used as a base class to avoid repetitious
definitions.
"""
#: optional Comment or Whitespace
_ws = r'(?:\s|//.*?\n|/[*].*?[*]/)+'
# The trailing ?, rather than *, avoids a geometric performance drop here.
#: only one /* */ style comment
_ws1 = r'\s*(?:/[*].*?[*]/\s*)?'
tokens = {
'whitespace': [
# preprocessor directives: without whitespace
('^#if\s+0', Comment.Preproc, 'if0'),
('^#', Comment.Preproc, 'macro'),
# or with whitespace
('^(' + _ws1 + r')(#if\s+0)',
bygroups(using(this), Comment.Preproc), 'if0'),
('^(' + _ws1 + ')(#)',
bygroups(using(this), Comment.Preproc), 'macro'),
(r'\n', Text),
(r'\s+', Text),
(r'\\\n', Text), # line continuation
(r'//(\n|[\w\W]*?[^\\]\n)', Comment.Single),
(r'/(\\\n)?[*][\w\W]*?[*](\\\n)?/', Comment.Multiline),
# Open until EOF, so no ending delimeter
(r'/(\\\n)?[*][\w\W]*', Comment.Multiline),
],
'statements': [
(r'(L?)(")', bygroups(String.Affix, String), 'string'),
(r"(L?)(')(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])(')",
bygroups(String.Affix, String.Char, String.Char, String.Char)),
(r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*', Number.Float),
(r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
(r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex),
(r'0[0-7]+[LlUu]*', Number.Oct),
(r'\d+[LlUu]*', Number.Integer),
(r'\*/', Error),
(r'[~!%^&*+=|?:<>/-]', Operator),
(r'[()\[\],.]', Punctuation),
(words(('asm', 'auto', 'break', 'case', 'const', 'continue',
'default', 'do', 'else', 'enum', 'extern', 'for', 'goto',
'if', 'register', 'restricted', 'return', 'sizeof',
'static', 'struct', 'switch', 'typedef', 'union',
'volatile', 'while'),
suffix=r'\b'), Keyword),
(r'(bool|int|long|float|short|double|char|unsigned|signed|void)\b',
Keyword.Type),
(words(('inline', '_inline', '__inline', 'naked', 'restrict',
'thread', 'typename'), suffix=r'\b'), Keyword.Reserved),
# Vector intrinsics
(r'(__m(128i|128d|128|64))\b', Keyword.Reserved),
# Microsoft-isms
(words((
'asm', 'int8', 'based', 'except', 'int16', 'stdcall', 'cdecl',
'fastcall', 'int32', 'declspec', 'finally', 'int64', 'try',
'leave', 'wchar_t', 'w64', 'unaligned', 'raise', 'noop',
'identifier', 'forceinline', 'assume'),
prefix=r'__', suffix=r'\b'), Keyword.Reserved),
(r'(true|false|NULL)\b', Name.Builtin),
(r'([a-zA-Z_]\w*)(\s*)(:)(?!:)', bygroups(Name.Label, Text, Punctuation)),
('[a-zA-Z_]\w*', Name),
],
'root': [
include('whitespace'),
# functions
(r'((?:[\w*\s])+?(?:\s|[*]))' # return arguments
r'([a-zA-Z_]\w*)' # method name
r'(\s*\([^;]*?\))' # signature
r'([^;{]*)(\{)',
bygroups(using(this), Name.Function, using(this), using(this),
Punctuation),
'function'),
# function declarations
(r'((?:[\w*\s])+?(?:\s|[*]))' # return arguments
r'([a-zA-Z_]\w*)' # method name
r'(\s*\([^;]*?\))' # signature
r'([^;]*)(;)',
bygroups(using(this), Name.Function, using(this), using(this),
Punctuation)),
default('statement'),
],
'statement': [
include('whitespace'),
include('statements'),
('[{}]', Punctuation),
(';', Punctuation, '#pop'),
],
'function': [
include('whitespace'),
include('statements'),
(';', Punctuation),
(r'\{', Punctuation, '#push'),
(r'\}', Punctuation, '#pop'),
],
'string': [
(r'"', String, '#pop'),
(r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|'
r'u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|[0-7]{1,3})', String.Escape),
(r'[^\\"\n]+', String), # all other characters
(r'\\\n', String), # line continuation
(r'\\', String), # stray backslash
],
'macro': [
(r'(include)(' + _ws1 + r')([^\n]+)',
bygroups(Comment.Preproc, Text, Comment.PreprocFile)),
(r'[^/\n]+', Comment.Preproc),
(r'/[*](.|\n)*?[*]/', Comment.Multiline),
(r'//.*?\n', Comment.Single, '#pop'),
(r'/', Comment.Preproc),
(r'(?<=\\)\n', Comment.Preproc),
(r'\n', Comment.Preproc, '#pop'),
],
'if0': [
(r'^\s*#if.*?(?<!\\)\n', Comment.Preproc, '#push'),
(r'^\s*#el(?:se|if).*\n', Comment.Preproc, '#pop'),
(r'^\s*#endif.*?(?<!\\)\n', Comment.Preproc, '#pop'),
(r'.*?\n', Comment),
]
}
stdlib_types = set((
'size_t', 'ssize_t', 'off_t', 'wchar_t', 'ptrdiff_t', 'sig_atomic_t', 'fpos_t',
'clock_t', 'time_t', 'va_list', 'jmp_buf', 'FILE', 'DIR', 'div_t', 'ldiv_t',
'mbstate_t', 'wctrans_t', 'wint_t', 'wctype_t'))
c99_types = set((
'_Bool', '_Complex', 'int8_t', 'int16_t', 'int32_t', 'int64_t', 'uint8_t',
'uint16_t', 'uint32_t', 'uint64_t', 'int_least8_t', 'int_least16_t',
'int_least32_t', 'int_least64_t', 'uint_least8_t', 'uint_least16_t',
'uint_least32_t', 'uint_least64_t', 'int_fast8_t', 'int_fast16_t', 'int_fast32_t',
'int_fast64_t', 'uint_fast8_t', 'uint_fast16_t', 'uint_fast32_t', 'uint_fast64_t',
'intptr_t', 'uintptr_t', 'intmax_t', 'uintmax_t'))
linux_types = set((
'clockid_t', 'cpu_set_t', 'cpumask_t', 'dev_t', 'gid_t', 'id_t', 'ino_t', 'key_t',
'mode_t', 'nfds_t', 'pid_t', 'rlim_t', 'sig_t', 'sighandler_t', 'siginfo_t',
'sigset_t', 'sigval_t', 'socklen_t', 'timer_t', 'uid_t'))
def __init__(self, **options):
self.stdlibhighlighting = get_bool_opt(options, 'stdlibhighlighting', True)
self.c99highlighting = get_bool_opt(options, 'c99highlighting', True)
self.platformhighlighting = get_bool_opt(options, 'platformhighlighting', True)
RegexLexer.__init__(self, **options)
def get_tokens_unprocessed(self, text):
for index, token, value in \
RegexLexer.get_tokens_unprocessed(self, text):
if token is Name:
if self.stdlibhighlighting and value in self.stdlib_types:
token = Keyword.Type
elif self.c99highlighting and value in self.c99_types:
token = Keyword.Type
elif self.platformhighlighting and value in self.linux_types:
token = Keyword.Type
yield index, token, value
class CLexer(CFamilyLexer):
"""
For C source code with preprocessor directives.
"""
name = 'C'
aliases = ['c']
filenames = ['*.c', '*.h', '*.idc']
mimetypes = ['text/x-chdr', 'text/x-csrc']
priority = 0.1
def analyse_text(text):
if re.search('^\s*#include [<"]', text, re.MULTILINE):
return 0.1
if re.search('^\s*#ifn?def ', text, re.MULTILINE):
return 0.1
class CppLexer(CFamilyLexer):
"""
For C++ source code with preprocessor directives.
"""
name = 'C++'
aliases = ['cpp', 'c++']
filenames = ['*.cpp', '*.hpp', '*.c++', '*.h++',
'*.cc', '*.hh', '*.cxx', '*.hxx',
'*.C', '*.H', '*.cp', '*.CPP']
mimetypes = ['text/x-c++hdr', 'text/x-c++src']
priority = 0.1
tokens = {
'statements': [
(words((
'catch', 'const_cast', 'delete', 'dynamic_cast', 'explicit',
'export', 'friend', 'mutable', 'namespace', 'new', 'operator',
'private', 'protected', 'public', 'reinterpret_cast',
'restrict', 'static_cast', 'template', 'this', 'throw', 'throws',
'try', 'typeid', 'typename', 'using', 'virtual',
'constexpr', 'nullptr', 'decltype', 'thread_local',
'alignas', 'alignof', 'static_assert', 'noexcept', 'override',
'final'), suffix=r'\b'), Keyword),
(r'char(16_t|32_t)\b', Keyword.Type),
(r'(class)(\s+)', bygroups(Keyword, Text), 'classname'),
# C++11 raw strings
(r'(R)(")([^\\()\s]{,16})(\()((?:.|\n)*?)(\)\3)(")',
bygroups(String.Affix, String, String.Delimiter, String.Delimiter,
String, String.Delimiter, String)),
# C++11 UTF-8/16/32 strings
(r'(u8|u|U)(")', bygroups(String.Affix, String), 'string'),
inherit,
],
'root': [
inherit,
# C++ Microsoft-isms
(words(('virtual_inheritance', 'uuidof', 'super', 'single_inheritance',
'multiple_inheritance', 'interface', 'event'),
prefix=r'__', suffix=r'\b'), Keyword.Reserved),
# Offload C++ extensions, http://offload.codeplay.com/
(r'__(offload|blockingoffload|outer)\b', Keyword.Pseudo),
],
'classname': [
(r'[a-zA-Z_]\w*', Name.Class, '#pop'),
# template specification
(r'\s*(?=>)', Text, '#pop'),
],
}
def analyse_text(text):
if re.search('#include <[a-z_]+>', text):
return 0.2
if re.search('using namespace ', text):
return 0.4

Some files were not shown because too many files have changed in this diff Show More