v8.3.2

HISTORY.rst

@@ -3,6 +3,131 @@ History
 -------
 
 
+8.3.2 (2018-10-06)
+++++++++++++++++++
+
+- Send buffered heartbeats to API every 30 seconds.
+
+
+8.3.1 (2018-10-05)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.4.1.
+- Send 50 offline heartbeats to API per request with 1 second delay in between.
+
+
+8.3.0 (2018-10-03)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.4.0.
+- Support logging coding activity to remote network drive files on Windows
+  platform by detecting UNC path from drive letter.
+  `wakatime#72 <https://github.com/wakatime/wakatime/issues/72>`_
+
+
+8.2.0 (2018-09-30)
+++++++++++++++++++
+
+- Prevent opening cmd window on Windows when running wakatime-cli.
+  `#91 <https://github.com/wakatime/sublime-wakatime/issues/91>`_
+- Upgrade wakatime-cli to v10.3.0.
+- Re-enable detecting projects from Subversion folder on Windows platform.
+- Prevent opening cmd window on Windows when detecting project from Subversion.
+- Run tests on Windows using Appveyor.
+
+
+8.1.2 (2018-09-20)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.2.4.
+- Default --sync-offline-activity to 100 instead of 5, so offline coding is
+  synced to dashboard faster.
+- Batch heartbeats in groups of 10 per api request.
+- New config hide_project_name and argument --hide-project-names for
+  obfuscating project names when sending coding activity to api.
+- Fix mispelled Gosu language.
+  `wakatime#137 <https://github.com/wakatime/wakatime/issues/137>`_
+- Remove metadata when hiding project or file names.
+- New --local-file argument to be used when --entity is a remote file.
+- New argument --sync-offline-activity for configuring the maximum offline
+  heartbeats to sync to the WakaTime API.
+
+
+8.1.1 (2018-04-26)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.2.1.
+- Force forward slash for file paths.
+- New --category argument.
+- New --exclude-unknown-project argument and corresponding config setting.
+- Support for project detection from git worktree folders.
+
+
+8.1.0 (2018-04-03)
+++++++++++++++++++
+
+- Prefer Python3 over Python2 when running wakatime-cli core.
+- Improve detection of Python3 on Ubuntu 17.10 platforms.
+
+
+8.0.8 (2018-03-15)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.1.3.
+- Smarter C vs C++ vs Objective-C language detection.
+
+
+8.0.7 (2018-03-15)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.1.2.
+- Detect dependencies from Swift, Objective-C, TypeScript and JavaScript files.
+- Categorize .mjs files as JavaScript.
+  `#wakatime121 <https://github.com/wakatime/wakatime/issues/121>`_
+- Detect dependencies from Elm, Haskell, Haxe, Kotlin, Rust, and Scala files.
+- Improved Matlab vs Objective-C language detection.
+  `#wakatime129 <https://github.com/wakatime/wakatime/issues/129>`_
+
+
+8.0.6 (2018-01-04)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.1.0.
+- Ability to only track folders containing a .wakatime-project file using new
+  include_only_with_project_file argument and config option.
+
+
+8.0.5 (2017-11-24)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.0.5.
+- Fix bug that caused heartbeats to be cached locally instead of sent to API.
+
+
+8.0.4 (2017-11-23)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.0.4.
+- Improve Java dependency detection.
+- Skip null or missing heartbeats from extra heartbeats argument.
+
+
+8.0.3 (2017-11-22)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.0.3.
+- Support saving unicode heartbeats when working offline.
+  `wakatime#112 <https://github.com/wakatime/wakatime/issues/112>`_
+
+
+8.0.2 (2017-11-15)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.0.2.
+- Limit bulk syncing to 5 heartbeats per request.
+  `wakatime#109 <https://github.com/wakatime/wakatime/issues/109>`_
+
+
 8.0.1 (2017-11-09)
 ++++++++++++++++++
 

WakaTime.py

@@ -7,7 +7,7 @@ Website:     https://wakatime.com/
 ==========================================================="""
 
 
-__version__ = '8.0.1'
+__version__ = '8.3.2'
 
 
 import sublime
@@ -18,6 +18,7 @@ import json
 import os
 import platform
 import re
+import subprocess
 import sys
 import time
 import threading
@@ -25,7 +26,7 @@ import traceback
 import urllib
 import webbrowser
 from datetime import datetime
-from subprocess import Popen, STDOUT, PIPE
+from subprocess import STDOUT, PIPE
 from zipfile import ZipFile
 try:
     import _winreg as winreg  # py2
@@ -42,6 +43,8 @@ except ImportError:
 
 is_py2 = (sys.version_info[0] == 2)
 is_py3 = (sys.version_info[0] == 3)
+is_win = platform.system() == 'Windows'
+
 
 if is_py2:
     def u(text):
@@ -91,8 +94,22 @@ else:
     ))
 
 
+class Popen(subprocess.Popen):
+    """Patched Popen to prevent opening cmd window on Windows platform."""
+
+    def __init__(self, *args, **kwargs):
+        startupinfo = kwargs.get('startupinfo')
+        if is_win or True:
+            try:
+                startupinfo = startupinfo or subprocess.STARTUPINFO()
+                startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+            except AttributeError:
+                pass
+        kwargs['startupinfo'] = startupinfo
+        super(Popen, self).__init__(*args, **kwargs)
+
+
 # globals
-HEARTBEAT_FREQUENCY = 2
 ST_VERSION = int(sublime.version())
 PLUGIN_DIR = os.path.dirname(os.path.realpath(__file__))
 API_CLIENT = os.path.join(PLUGIN_DIR, 'packages', 'wakatime', 'cli.py')
@@ -103,8 +120,11 @@ LAST_HEARTBEAT = {
     'file': None,
     'is_write': False,
 }
+LAST_HEARTBEAT_SENT_AT = 0
 PYTHON_LOCATION = None
 HEARTBEATS = queue.Queue()
+HEARTBEAT_FREQUENCY = 2  # minutes between logging heartbeat when editing same file
+SEND_BUFFER_SECONDS = 30  # seconds between sending buffered heartbeats to API
 
 
 # Log Levels
@@ -305,13 +325,15 @@ def find_python_from_registry(location, reg=None):
     return val
 
 
-def find_python_in_folder(folder, headless=True):
+def find_python_in_folder(folder, python3=True, headless=True):
     pattern = re.compile(r'\d+\.\d+')
 
     path = 'python'
-    if folder is not None:
+    if folder:
         path = os.path.realpath(os.path.join(folder, 'python'))
-    if headless:
+    if python3:
+        path = u(path) + u('3')
+    elif headless:
         path = u(path) + u('w')
     log(DEBUG, u('Looking for Python at: {0}').format(u(path)))
     try:
@@ -325,9 +347,13 @@ def find_python_in_folder(folder, headless=True):
     except:
         log(DEBUG, u(sys.exc_info()[1]))
 
-    if headless:
-        path = find_python_in_folder(folder, headless=False)
-        if path is not None:
+    if python3:
+        path = find_python_in_folder(folder, python3=False, headless=headless)
+        if path:
+            return path
+    elif headless:
+        path = find_python_in_folder(folder, python3=python3, headless=False)
+        if path:
             return path
 
     return None
@@ -427,11 +453,18 @@ def append_heartbeat(entity, timestamp, is_write, view, project, folders):
     }
 
     # process the queue of heartbeats in the future
-    seconds = 4
-    set_timeout(process_queue, seconds)
+    set_timeout(lambda: process_queue(timestamp), SEND_BUFFER_SECONDS)
 
 
-def process_queue():
+def process_queue(timestamp):
+    global LAST_HEARTBEAT_SENT_AT
+
+    # Prevent sending heartbeats more often than SEND_BUFFER_SECONDS
+    now = int(time.time())
+    if timestamp != LAST_HEARTBEAT['time'] and LAST_HEARTBEAT_SENT_AT > now - SEND_BUFFER_SECONDS:
+        return
+    LAST_HEARTBEAT_SENT_AT = now
+
     try:
         heartbeat = HEARTBEATS.get_nowait()
     except queue.Empty:
@@ -536,19 +569,16 @@ class SendHeartbeatsThread(threading.Thread):
             if self.has_extra_heartbeats:
                 cmd.append('--extra-heartbeats')
                 stdin = PIPE
-                extra_heartbeats = [self.build_heartbeat(**x) for x in self.extra_heartbeats]
-                extra_heartbeats = json.dumps(extra_heartbeats)
+                extra_heartbeats = json.dumps([self.build_heartbeat(**x) for x in self.extra_heartbeats])
+                inp = "{0}\n".format(extra_heartbeats).encode('utf-8')
             else:
                 extra_heartbeats = None
                 stdin = None
+                inp = None
 
             log(DEBUG, ' '.join(obfuscate_apikey(cmd)))
             try:
                 process = Popen(cmd, stdin=stdin, stdout=PIPE, stderr=STDOUT)
-                inp = None
-                if self.has_extra_heartbeats:
-                    inp = "{0}\n".format(extra_heartbeats)
-                    inp = inp.encode('utf-8')
                 output, err = process.communicate(input=inp)
                 output = u(output)
                 retcode = process.poll()

packages/wakatime/__about__.py

@@ -1,7 +1,7 @@
 __title__ = 'wakatime'
 __description__ = 'Common interface to the WakaTime api.'
 __url__ = 'https://github.com/wakatime/wakatime'
-__version_info__ = ('10', '0', '1')
+__version_info__ = ('10', '4', '1')
 __version__ = '.'.join(__version_info__)
 __author__ = 'Alan Hamlett'
 __author_email__ = 'alan@wakatime.com'

packages/wakatime/api.py

@@ -30,7 +30,7 @@ log = logging.getLogger('WakaTime')
 
 try:
     from .packages import requests
-except ImportError:
+except ImportError:  # pragma: nocover
     log.traceback(logging.ERROR)
     print(traceback.format_exc())
     log.error('Please upgrade Python to the latest version.')
@@ -138,50 +138,59 @@ def send_heartbeats(heartbeats, args, configs, use_ntlm_proxy=False):
     else:
         code = response.status_code if response is not None else None
         content = response.text if response is not None else None
-        try:
-            results = response.json() if response is not None else []
-        except:
-            if log.isEnabledFor(logging.DEBUG):
-                log.traceback(logging.WARNING)
-            results = []
-        if code == requests.codes.created or code == requests.codes.accepted:
-            log.debug({
-                'response_code': code,
-            })
-
-            for i in range(len(results)):
-                if len(heartbeats) <= i:
-                    log.debug('Results from server do not match heartbeats sent.')
-                    break
-
-                try:
-                    c = results[i][1]
-                except:
-                    c = 0
-                try:
-                    text = json.dumps(results[i][0])
-                except:
-                    if log.isEnabledFor(logging.DEBUG):
-                        log.traceback(logging.WARNING)
-                    text = ''
-                handle_result([heartbeats[i]], c, text, args, configs)
 
+        if _success(code):
+            results = _get_results(response)
+            _process_server_results(heartbeats, code, content, results, args, configs)
             session_cache.save(session)
             return SUCCESS
+        else:
+            log.debug({
+                'response_code': code,
+                'response_text': content,
+            })
 
         if should_try_ntlm:
             return send_heartbeats(heartbeats, args, configs, use_ntlm_proxy=True)
-        else:
-            handle_result(heartbeats, code, content, args, configs)
+
+        _handle_unsent_heartbeats(heartbeats, code, content, args, configs)
 
     session_cache.delete()
     return AUTH_ERROR if code == 401 else API_ERROR
 
 
-def handle_result(h, code, content, args, configs):
-    if code == requests.codes.created or code == requests.codes.accepted:
-        return
+def _process_server_results(heartbeats, code, content, results, args, configs):
+    log.debug({
+        'response_code': code,
+        'results': results,
+    })
 
+    for i in range(len(results)):
+        if len(heartbeats) <= i:
+            log.warn('Results from api not matching heartbeats sent.')
+            break
+
+        try:
+            c = results[i][1]
+        except:
+            log.traceback(logging.WARNING)
+            c = 0
+        try:
+            text = json.dumps(results[i][0])
+        except:
+            log.traceback(logging.WARNING)
+            text = ''
+        if not _success(c):
+            _handle_unsent_heartbeats([heartbeats[i]], c, text, args, configs)
+
+    leftover = len(heartbeats) - len(results)
+    if leftover > 0:
+        log.warn('Missing {0} results from api.'.format(leftover))
+        start = len(heartbeats) - leftover
+        _handle_unsent_heartbeats(heartbeats[start:], code, content, args, configs)
+
+
+def _handle_unsent_heartbeats(heartbeats, code, content, args, configs):
     if args.offline:
         if code == 400:
             log.error({
@@ -195,9 +204,23 @@ def handle_result(h, code, content, args, configs):
                     'response_content': content,
                 })
             queue = Queue(args, configs)
-            queue.push_many(h)
+            queue.push_many(heartbeats)
     else:
         log.error({
             'response_code': code,
             'response_content': content,
         })
+
+
+def _get_results(response):
+    results = []
+    if response is not None:
+        try:
+            results = response.json()['responses']
+        except:
+            log.traceback(logging.WARNING)
+    return results
+
+
+def _success(code):
+    return code == requests.codes.created or code == requests.codes.accepted

packages/wakatime/arguments.py

@@ -20,7 +20,7 @@ import traceback
 from .__about__ import __version__
 from .compat import basestring
 from .configs import parseConfigFile
-from .constants import AUTH_ERROR
+from .constants import AUTH_ERROR, DEFAULT_SYNC_OFFLINE_ACTIVITY
 from .packages import argparse
 
 
@@ -52,86 +52,155 @@ def parse_arguments():
     """
 
     # define supported command line arguments
-    parser = argparse.ArgumentParser(
-            description='Common interface for the WakaTime api.')
+    parser = argparse.ArgumentParser(description='Common interface for the ' +
+                                                 'WakaTime api.')
     parser.add_argument('--entity', dest='entity', metavar='FILE',
-            action=FileAction,
-            help='absolute path to file for the heartbeat; can also be a '+
-                 'url, domain, or app when --entity-type is not file')
+                        action=FileAction,
+                        help='Absolute path to file for the heartbeat. Can ' +
+                             'also be a url, domain or app when ' +
+                             '--entity-type is not file.')
     parser.add_argument('--file', dest='file', action=FileAction,
-            help=argparse.SUPPRESS)
+                        help=argparse.SUPPRESS)
     parser.add_argument('--key', dest='key', action=StoreWithoutQuotes,
-            help='your wakatime api key; uses api_key from '+
-                '~/.wakatime.cfg by default')
-    parser.add_argument('--write', dest='is_write',
-            action='store_true',
-            help='when set, tells api this heartbeat was triggered from '+
-                 'writing to a file')
+                        help='Your wakatime api key; uses api_key from ' +
+                             '~/.wakatime.cfg by default.')
+    parser.add_argument('--write', dest='is_write', action='store_true',
+                        help='When set, tells api this heartbeat was ' +
+                             'triggered from writing to a file.')
     parser.add_argument('--plugin', dest='plugin', action=StoreWithoutQuotes,
-            help='optional text editor plugin name and version '+
-                 'for User-Agent header')
+                        help='Optional text editor plugin name and version ' +
+                             'for User-Agent header.')
     parser.add_argument('--time', dest='timestamp', metavar='time',
-            type=float, action=StoreWithoutQuotes,
-            help='optional floating-point unix epoch timestamp; '+
-                 'uses current time by default')
+                        type=float, action=StoreWithoutQuotes,
+                        help='Optional floating-point unix epoch timestamp. ' +
+                             'Uses current time by default.')
     parser.add_argument('--lineno', dest='lineno', action=StoreWithoutQuotes,
-            help='optional line number; current line being edited')
-    parser.add_argument('--cursorpos', dest='cursorpos', action=StoreWithoutQuotes,
-            help='optional cursor position in the current file')
-    parser.add_argument('--entity-type', dest='entity_type', action=StoreWithoutQuotes,
-            help='entity type for this heartbeat. can be one of "file", '+
-                 '"domain", or "app"; defaults to file.')
+                        help='Optional line number. This is the current ' +
+                             'line being edited.')
+    parser.add_argument('--cursorpos', dest='cursorpos',
+                        action=StoreWithoutQuotes,
+                        help='Optional cursor position in the current file.')
+    parser.add_argument('--entity-type', dest='entity_type',
+                        action=StoreWithoutQuotes,
+                        help='Entity type for this heartbeat. Can be ' +
+                             '"file", "domain" or "app". Defaults to "file".')
+    parser.add_argument('--category', dest='category',
+                        action=StoreWithoutQuotes,
+                        help='Category of this heartbeat activity. Can be ' +
+                             '"coding", "building", "indexing", ' +
+                             '"debugging", "running tests", ' +
+                             '"manual testing", "browsing", ' +
+                             '"code reviewing" or "designing". ' +
+                             'Defaults to "coding".')
     parser.add_argument('--proxy', dest='proxy', action=StoreWithoutQuotes,
-                        help='optional proxy configuration. Supports HTTPS '+
-                        'and SOCKS proxies. For example: '+
-                        'https://user:pass@host:port or '+
-                        'socks5://user:pass@host:port or ' +
-                        'domain\\user:pass')
+                        help='Optional proxy configuration. Supports HTTPS '+
+                             'and SOCKS proxies. For example: '+
+                             'https://user:pass@host:port or '+
+                             'socks5://user:pass@host:port or ' +
+                             'domain\\user:pass')
     parser.add_argument('--no-ssl-verify', dest='nosslverify',
                         action='store_true',
-                        help='disables SSL certificate verification for HTTPS '+
-                        'requests. By default, SSL certificates are verified.')
+                        help='Disables SSL certificate verification for HTTPS '+
+                             'requests. By default, SSL certificates are ' +
+                             'verified.')
     parser.add_argument('--project', dest='project', action=StoreWithoutQuotes,
-            help='optional project name')
-    parser.add_argument('--alternate-project', dest='alternate_project', action=StoreWithoutQuotes,
-            help='optional alternate project name; auto-discovered project '+
-                 'takes priority')
-    parser.add_argument('--alternate-language', dest='alternate_language', action=StoreWithoutQuotes,
-            help=argparse.SUPPRESS)
-    parser.add_argument('--language', dest='language', action=StoreWithoutQuotes,
-            help='optional language name; if valid, takes priority over '+
-                 'auto-detected language')
-    parser.add_argument('--hostname', dest='hostname', action=StoreWithoutQuotes, help='hostname of '+
-                        'current machine.')
-    parser.add_argument('--disableoffline', dest='offline',
-            action='store_false',
-            help='disables offline time logging instead of queuing logged time')
+                        help='Optional project name.')
+    parser.add_argument('--alternate-project', dest='alternate_project',
+                        action=StoreWithoutQuotes,
+                        help='Optional alternate project name. ' +
+                             'Auto-discovered project takes priority.')
+    parser.add_argument('--alternate-language', dest='alternate_language',
+                        action=StoreWithoutQuotes,
+                        help=argparse.SUPPRESS)
+    parser.add_argument('--language', dest='language',
+                        action=StoreWithoutQuotes,
+                        help='Optional language name. If valid, takes ' +
+                             'priority over auto-detected language.')
+    parser.add_argument('--local-file', dest='local_file', metavar='FILE',
+                        action=FileAction,
+                        help='Absolute path to local file for the ' +
+                             'heartbeat. When --entity is a remote file, ' +
+                             'this local file will be used for stats and ' +
+                             'just the value of --entity sent with heartbeat.')
+    parser.add_argument('--hostname', dest='hostname',
+                        action=StoreWithoutQuotes,
+                        help='Hostname of current machine.')
+    parser.add_argument('--disable-offline', dest='offline',
+                        action='store_false',
+                        help='Disables offline time logging instead of ' +
+                             'queuing logged time.')
+    parser.add_argument('--disableoffline', dest='offline_deprecated',
+                        action='store_true',
+                        help=argparse.SUPPRESS)
+    parser.add_argument('--hide-file-names', dest='hide_file_names',
+                        action='store_true',
+                        help='Obfuscate filenames. Will not send file names ' +
+                             'to api.')
+    parser.add_argument('--hide-filenames', dest='hide_filenames',
+                        action='store_true',
+                        help=argparse.SUPPRESS)
     parser.add_argument('--hidefilenames', dest='hidefilenames',
-            action='store_true',
-            help='obfuscate file names; will not send file names to api')
+                        action='store_true',
+                        help=argparse.SUPPRESS)
+    parser.add_argument('--hide-project-names', dest='hide_project_names',
+                        action='store_true',
+                        help='Obfuscate project names. When a project ' +
+                             'folder is detected instead of using the ' +
+                             'folder name as the project, a ' +
+                             '.wakatime-project file is created with a ' +
+                             'random project name.')
     parser.add_argument('--exclude', dest='exclude', action='append',
-            help='filename patterns to exclude from logging; POSIX regex '+
-                 'syntax; can be used more than once')
+                        help='Filename patterns to exclude from logging. ' +
+                             'POSIX regex syntax. Can be used more than once.')
+    parser.add_argument('--exclude-unknown-project',
+                        dest='exclude_unknown_project', action='store_true',
+                        help='When set, any activity where the project ' +
+                             'cannot be detected will be ignored.')
     parser.add_argument('--include', dest='include', action='append',
-            help='filename patterns to log; when used in combination with '+
-                 '--exclude, files matching include will still be logged; '+
-                 'POSIX regex syntax; can be used more than once')
+                        help='Filename patterns to log. When used in ' +
+                             'combination with --exclude, files matching ' +
+                             'include will still be logged. POSIX regex ' +
+                             'syntax. Can be used more than once.')
+    parser.add_argument('--include-only-with-project-file',
+                        dest='include_only_with_project_file',
+                        action='store_true',
+                        help='Disables tracking folders unless they contain ' +
+                             'a .wakatime-project file. Defaults to false.')
     parser.add_argument('--ignore', dest='ignore', action='append',
-            help=argparse.SUPPRESS)
+                        help=argparse.SUPPRESS)
     parser.add_argument('--extra-heartbeats', dest='extra_heartbeats',
-            action='store_true',
-            help='reads extra heartbeats from STDIN as a JSON array until EOF')
+                        action='store_true',
+                        help='Reads extra heartbeats from STDIN as a JSON ' +
+                             'array until EOF.')
+    parser.add_argument('--log-file', dest='log_file',
+                        action=StoreWithoutQuotes,
+                        help='Defaults to ~/.wakatime.log.')
     parser.add_argument('--logfile', dest='logfile', action=StoreWithoutQuotes,
-            help='defaults to ~/.wakatime.log')
-    parser.add_argument('--apiurl', dest='api_url', action=StoreWithoutQuotes,
-            help='heartbeats api url; for debugging with a local server')
-    parser.add_argument('--timeout', dest='timeout', type=int, action=StoreWithoutQuotes,
-            help='number of seconds to wait when sending heartbeats to api; '+
-                 'defaults to 60 seconds')
+                        help=argparse.SUPPRESS)
+    parser.add_argument('--api-url', dest='api_url', action=StoreWithoutQuotes,
+                        help='Heartbeats api url. For debugging with a ' +
+                             'local server.')
+    parser.add_argument('--apiurl', dest='apiurl', action=StoreWithoutQuotes,
+                        help=argparse.SUPPRESS)
+    parser.add_argument('--timeout', dest='timeout', type=int,
+                        action=StoreWithoutQuotes,
+                        help='Number of seconds to wait when sending ' +
+                             'heartbeats to api. Defaults to 60 seconds.')
+    parser.add_argument('--sync-offline-activity',
+                        dest='sync_offline_activity',
+                        action=StoreWithoutQuotes,
+                        help='Amount of offline activity to sync from your ' +
+                             'local ~/.wakatime.db sqlite3 file to your ' +
+                             'WakaTime Dashboard before exiting. Can be ' +
+                             '"none" or a positive integer number. Defaults ' +
+                             'to 5, meaning for every heartbeat sent while ' +
+                             'online 5 offline heartbeats are synced. Can ' +
+                             'be used without --entity to only sync offline ' +
+                             'activity without generating new heartbeats.')
     parser.add_argument('--config', dest='config', action=StoreWithoutQuotes,
-            help='defaults to ~/.wakatime.cfg')
+                        help='Defaults to ~/.wakatime.cfg.')
     parser.add_argument('--verbose', dest='verbose', action='store_true',
-            help='turns on debug messages in log file')
+                        help='Turns on debug messages in log file.')
     parser.add_argument('--version', action='version', version=__version__)
 
     # parse command line arguments
@@ -158,23 +227,34 @@ def parse_arguments():
             args.key = default_key
         else:
             try:
-                parser.error('Missing api key. Find your api key from wakatime.com/settings.')
+                parser.error('Missing api key. Find your api key from wakatime.com/settings/api-key.')
             except SystemExit:
                 raise SystemExit(AUTH_ERROR)
 
     is_valid = not not re.match(r'^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$', args.key, re.I)
     if not is_valid:
         try:
-            parser.error('Invalid api key. Find your api key from wakatime.com/settings.')
+            parser.error('Invalid api key. Find your api key from wakatime.com/settings/api-key.')
         except SystemExit:
             raise SystemExit(AUTH_ERROR)
 
     if not args.entity:
         if args.file:
             args.entity = args.file
-        else:
+        elif not args.sync_offline_activity or args.sync_offline_activity == 'none':
             parser.error('argument --entity is required')
 
+    if not args.sync_offline_activity:
+        args.sync_offline_activity = DEFAULT_SYNC_OFFLINE_ACTIVITY
+    if args.sync_offline_activity == 'none':
+        args.sync_offline_activity = 0
+    try:
+        args.sync_offline_activity = int(args.sync_offline_activity)
+        if args.sync_offline_activity < 0:
+            raise Exception('Error')
+    except:
+        parser.error('argument --sync-offline-activity must be "none" or an integer number')
+
     if not args.language and args.alternate_language:
         args.language = args.alternate_language
 
@@ -194,6 +274,8 @@ def parse_arguments():
                     args.exclude.append(pattern)
         except TypeError:  # pragma: nocover
             pass
+    if not args.include_only_with_project_file and configs.has_option('settings', 'include_only_with_project_file'):
+        args.include_only_with_project_file = configs.get('settings', 'include_only_with_project_file')
     if not args.include:
         args.include = []
     if configs.has_option('settings', 'include'):
@@ -203,18 +285,12 @@ def parse_arguments():
                     args.include.append(pattern)
         except TypeError:  # pragma: nocover
             pass
-    if args.hidefilenames:
-        args.hidefilenames = ['.*']
-    else:
-        args.hidefilenames = []
-        if configs.has_option('settings', 'hidefilenames'):
-            option = configs.get('settings', 'hidefilenames')
-            if option.strip().lower() == 'true':
-                args.hidefilenames = ['.*']
-            elif option.strip().lower() != 'false':
-                for pattern in option.split("\n"):
-                    if pattern.strip() != '':
-                        args.hidefilenames.append(pattern)
+    if not args.exclude_unknown_project and configs.has_option('settings', 'exclude_unknown_project'):
+        args.exclude_unknown_project = configs.getboolean('settings', 'exclude_unknown_project')
+    boolean_or_list('hide_file_names', args, configs, alternative_names=['hide_filenames', 'hidefilenames'])
+    boolean_or_list('hide_project_names', args, configs, alternative_names=['hide_projectnames', 'hideprojectnames'])
+    if args.offline_deprecated:
+        args.offline = False
     if args.offline and configs.has_option('settings', 'offline'):
         args.offline = configs.getboolean('settings', 'offline')
     if not args.proxy and configs.has_option('settings', 'proxy'):
@@ -235,11 +311,15 @@ def parse_arguments():
         args.verbose = configs.getboolean('settings', 'verbose')
     if not args.verbose and configs.has_option('settings', 'debug'):
         args.verbose = configs.getboolean('settings', 'debug')
-    if not args.logfile and configs.has_option('settings', 'logfile'):
-        args.logfile = configs.get('settings', 'logfile')
-    if not args.logfile and os.environ.get('WAKATIME_HOME'):
+    if not args.log_file and args.logfile:
+        args.log_file = args.logfile
+    if not args.log_file and configs.has_option('settings', 'log_file'):
+        args.log_file = configs.get('settings', 'log_file')
+    if not args.log_file and os.environ.get('WAKATIME_HOME'):
         home = os.environ.get('WAKATIME_HOME')
-        args.logfile = os.path.join(os.path.expanduser(home), '.wakatime.log')
+        args.log_file = os.path.join(os.path.expanduser(home), '.wakatime.log')
+    if not args.api_url and args.apiurl:
+        args.api_url = args.apiurl
     if not args.api_url and configs.has_option('settings', 'api_url'):
         args.api_url = configs.get('settings', 'api_url')
     if not args.timeout and configs.has_option('settings', 'timeout'):
@@ -249,3 +329,30 @@ def parse_arguments():
             print(traceback.format_exc())
 
     return args, configs
+
+
+def boolean_or_list(config_name, args, configs, alternative_names=[]):
+    """Get a boolean or list of regexes from args and configs."""
+
+    # when argument flag present, set to wildcard regex
+    for key in alternative_names:
+        if hasattr(args, key) and getattr(args, key):
+            setattr(args, config_name, ['.*'])
+            return
+
+    setattr(args, config_name, [])
+
+    option = None
+    alternative_names.insert(0, config_name)
+    for key in alternative_names:
+        if configs.has_option('settings', key):
+            option = configs.get('settings', key)
+            break
+
+    if option is not None:
+        if option.strip().lower() == 'true':
+            setattr(args, config_name, ['.*'])
+        elif option.strip().lower() != 'false':
+            for pattern in option.split("\n"):
+                if pattern.strip() != '':
+                    getattr(args, config_name).append(pattern)

packages/wakatime/compat.py

@@ -11,11 +11,15 @@
 
 
 import codecs
+import os
+import platform
+import subprocess
 import sys
 
 
 is_py2 = (sys.version_info[0] == 2)
 is_py3 = (sys.version_info[0] == 3)
+is_win = platform.system() == 'Windows'
 
 
 if is_py2:  # pragma: nocover
@@ -96,5 +100,23 @@ except ImportError:  # pragma: nocover
 
 try:
     from .packages import simplejson as json
-except (ImportError, SyntaxError):
+except (ImportError, SyntaxError):  # pragma: nocover
     import json
+
+
+class Popen(subprocess.Popen):
+    """Patched Popen to prevent opening cmd window on Windows platform."""
+
+    def __init__(self, *args, **kwargs):
+        startupinfo = kwargs.get('startupinfo')
+        if is_win or True:
+            try:
+                startupinfo = startupinfo or subprocess.STARTUPINFO()
+                startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+            except AttributeError:
+                pass
+        kwargs['startupinfo'] = startupinfo
+        if 'env' not in kwargs:
+            kwargs['env'] = os.environ.copy()
+            kwargs['env']['LANG'] = 'en-US' if is_win else 'en_US.UTF-8'
+        subprocess.Popen.__init__(self, *args, **kwargs)

packages/wakatime/configs.py

@@ -21,7 +21,7 @@ from .constants import CONFIG_FILE_PARSE_ERROR
 
 try:
     import configparser
-except ImportError:
+except ImportError:  # pragma: nocover
     from .packages import configparser
 
 

packages/wakatime/constants.py

@@ -45,3 +45,12 @@ Files larger than this in bytes will not have a line count stat for performance.
 Default is 2MB.
 """
 MAX_FILE_SIZE_SUPPORTED = 2000000
+
+""" Default limit of number of offline heartbeats to sync before exiting."""
+DEFAULT_SYNC_OFFLINE_ACTIVITY = 100
+
+""" Number of heartbeats per api request.
+Even when sending more heartbeats, this is the number of heartbeats sent per
+individual https request to the WakaTime API.
+"""
+HEARTBEATS_PER_REQUEST = 50

packages/wakatime/dependencies/__init__.py

@@ -106,8 +106,8 @@ class DependencyParser(object):
         self.lexer = lexer
 
         if self.lexer:
-            module_name = self.lexer.__module__.rsplit('.', 1)[-1]
-            class_name = self.lexer.__class__.__name__.replace('Lexer', 'Parser', 1)
+            module_name = self.root_lexer.__module__.rsplit('.', 1)[-1]
+            class_name = self.root_lexer.__class__.__name__.replace('Lexer', 'Parser', 1)
         else:
             module_name = 'unknown'
             class_name = 'UnknownParser'
@@ -121,6 +121,12 @@ class DependencyParser(object):
         except ImportError:
             log.debug('Parsing dependencies not supported for {0}.{1}'.format(module_name, class_name))
 
+    @property
+    def root_lexer(self):
+        if hasattr(self.lexer, 'root_lexer'):
+            return self.lexer.root_lexer
+        return self.lexer
+
     def parse(self):
         if self.parser:
             plugin = self.parser(self.source_file, lexer=self.lexer)

packages/wakatime/dependencies/c_cpp.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
-    wakatime.languages.c_cpp
-    ~~~~~~~~~~~~~~~~~~~~~~~~
+    wakatime.dependencies.c_cpp
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Parse dependencies from C++ code.
 

packages/wakatime/dependencies/data.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
-    wakatime.languages.data
-    ~~~~~~~~~~~~~~~~~~~~~~~
+    wakatime.dependencies.data
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Parse dependencies from data files.
 

packages/wakatime/dependencies/dotnet.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
-    wakatime.languages.dotnet
-    ~~~~~~~~~~~~~~~~~~~~~~~~~
+    wakatime.dependencies.dotnet
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Parse dependencies from .NET code.
 

packages/wakatime/dependencies/elm.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.elm
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Elm code.
+
+    :copyright: (c) 2018 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+
+
+class ElmParser(TokenParser):
+    state = None
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        elif self.partial(token) == 'Class':
+            self._process_class(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_namespace(self, token, content):
+        self.state = content.strip()
+
+    def _process_class(self, token, content):
+        if self.state == 'import':
+            self.append(self._format(content))
+
+    def _process_text(self, token, content):
+        pass
+
+    def _process_other(self, token, content):
+        self.state = None
+
+    def _format(self, content):
+        return content.strip().split('.')[0].strip()

packages/wakatime/dependencies/go.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
-    wakatime.languages.go
-    ~~~~~~~~~~~~~~~~~~~~~
+    wakatime.dependencies.go
+    ~~~~~~~~~~~~~~~~~~~~~~~~
 
     Parse dependencies from Go code.
 

packages/wakatime/dependencies/haskell.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.haskell
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Haskell code.
+
+    :copyright: (c) 2018 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+
+
+class HaskellParser(TokenParser):
+    state = None
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Reserved':
+            self._process_reserved(token, content)
+        elif self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        elif self.partial(token) == 'Keyword':
+            self._process_keyword(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_reserved(self, token, content):
+        self.state = content.strip()
+
+    def _process_namespace(self, token, content):
+        if self.state == 'import':
+            self.append(self._format(content))
+
+    def _process_keyword(self, token, content):
+        if self.state != 'import' or content.strip() != 'qualified':
+            self.state = None
+
+    def _process_text(self, token, content):
+        pass
+
+    def _process_other(self, token, content):
+        self.state = None
+
+    def _format(self, content):
+        return content.strip().split('.')[0].strip()

packages/wakatime/dependencies/haxe.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.haxe
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Haxe code.
+
+    :copyright: (c) 2018 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+
+
+class HaxeParser(TokenParser):
+    exclude = [
+        r'^haxe$',
+    ]
+    state = None
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_namespace(self, token, content):
+        if self.state == 'import':
+            self.append(self._format(content))
+            self.state = None
+        else:
+            self.state = content
+
+    def _process_text(self, token, content):
+        pass
+
+    def _process_other(self, token, content):
+        self.state = None
+
+    def _format(self, content):
+        return content.strip()

packages/wakatime/dependencies/html.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 """
-    wakatime.languages.templates
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    wakatime.dependencies.html
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    Parse dependencies from Templates.
+    Parse dependencies from HTML.
 
     :copyright: (c) 2014 Alan Hamlett.
     :license: BSD, see LICENSE for more details.
@@ -69,7 +69,7 @@ KEYWORDS = [
 ]
 
 
-class HtmlDjangoParser(TokenParser):
+class HtmlParser(TokenParser):
     tags = []
     opening_tag = False
     getting_attrs = False
@@ -141,63 +141,3 @@ class HtmlDjangoParser(TokenParser):
             elif content.startswith('"') or content.startswith("'"):
                 if self.current_attr_value is None:
                     self.current_attr_value = content
-
-
-class VelocityHtmlParser(HtmlDjangoParser):
-    pass
-
-
-class MyghtyHtmlParser(HtmlDjangoParser):
-    pass
-
-
-class MasonParser(HtmlDjangoParser):
-    pass
-
-
-class MakoHtmlParser(HtmlDjangoParser):
-    pass
-
-
-class CheetahHtmlParser(HtmlDjangoParser):
-    pass
-
-
-class HtmlGenshiParser(HtmlDjangoParser):
-    pass
-
-
-class RhtmlParser(HtmlDjangoParser):
-    pass
-
-
-class HtmlPhpParser(HtmlDjangoParser):
-    pass
-
-
-class HtmlSmartyParser(HtmlDjangoParser):
-    pass
-
-
-class EvoqueHtmlParser(HtmlDjangoParser):
-    pass
-
-
-class ColdfusionHtmlParser(HtmlDjangoParser):
-    pass
-
-
-class LassoHtmlParser(HtmlDjangoParser):
-    pass
-
-
-class HandlebarsHtmlParser(HtmlDjangoParser):
-    pass
-
-
-class YamlJinjaParser(HtmlDjangoParser):
-    pass
-
-
-class TwigHtmlParser(HtmlDjangoParser):
-    pass

packages/wakatime/dependencies/javascript.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.javascript
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from JavaScript code.
+
+    :copyright: (c) 2018 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import re
+
+from . import TokenParser
+
+
+class JavascriptParser(TokenParser):
+    state = None
+    extension = re.compile(r'\.\w{1,4}$')
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Reserved':
+            self._process_reserved(token, content)
+        elif self.partial(token) == 'Single':
+            self._process_string(token, content)
+        elif self.partial(token) == 'Punctuation':
+            self._process_punctuation(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_reserved(self, token, content):
+        if self.state is None:
+            self.state = content
+
+    def _process_string(self, token, content):
+        if self.state == 'import':
+            self.append(self._format_module(content))
+        self.state = None
+
+    def _process_punctuation(self, token, content):
+        if content == ';':
+            self.state = None
+
+    def _process_other(self, token, content):
+        pass
+
+    def _format_module(self, content):
+        content = content.strip().strip('"').strip("'").strip()
+        content = content.split('/')[-1].split('\\')[-1]
+        content = self.extension.sub('', content, count=1)
+        return content
+
+
+class TypeScriptParser(JavascriptParser):
+    pass

packages/wakatime/dependencies/jvm.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
-    wakatime.languages.java
-    ~~~~~~~~~~~~~~~~~~~~~~~
+    wakatime.dependencies.java
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Parse dependencies from Java code.
 
@@ -43,7 +43,7 @@ class JavaParser(TokenParser):
             self._process_other(token, content)
 
     def _process_namespace(self, token, content):
-        if u(content) == u('import'):
+        if u(content).split() and u(content).split()[0] == u('import'):
             self.state = 'import'
 
         elif self.state == 'import':
@@ -94,3 +94,89 @@ class JavaParser(TokenParser):
 
     def _process_other(self, token, content):
         pass
+
+
+class KotlinParser(TokenParser):
+    state = None
+    exclude = [
+        r'^java\.',
+    ]
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Keyword':
+            self._process_keyword(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        elif self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_keyword(self, token, content):
+        self.state = content
+
+    def _process_text(self, token, content):
+        pass
+
+    def _process_namespace(self, token, content):
+        if self.state == 'import':
+            self.append(self._format(content))
+        self.state = None
+
+    def _process_other(self, token, content):
+        self.state = None
+
+    def _format(self, content):
+        content = content.split(u('.'))
+
+        if content[-1] == u('*'):
+            content = content[:len(content) - 1]
+
+        if len(content) == 0:
+            return None
+
+        if len(content) == 1:
+            return content[0]
+
+        return u('.').join(content[:2])
+
+
+class ScalaParser(TokenParser):
+    state = None
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Keyword':
+            self._process_keyword(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        elif self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_keyword(self, token, content):
+        self.state = content
+
+    def _process_text(self, token, content):
+        pass
+
+    def _process_namespace(self, token, content):
+        if self.state == 'import':
+            self.append(self._format(content))
+        self.state = None
+
+    def _process_other(self, token, content):
+        self.state = None
+
+    def _format(self, content):
+        return content.strip().lstrip('__root__').strip('_').strip('.')

packages/wakatime/dependencies/objective.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.objective
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Objective-C and Swift code.
+
+    :copyright: (c) 2018 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import re
+
+from . import TokenParser
+
+
+class SwiftParser(TokenParser):
+    state = None
+    exclude = [
+        r'^foundation$',
+    ]
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Declaration':
+            self._process_declaration(token, content)
+        elif self.partial(token) == 'Class':
+            self._process_class(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_declaration(self, token, content):
+        if self.state is None:
+            self.state = content
+
+    def _process_class(self, token, content):
+        if self.state == 'import':
+            self.append(content)
+        self.state = None
+
+    def _process_other(self, token, content):
+        pass
+
+
+class ObjectiveCParser(TokenParser):
+    state = None
+    extension = re.compile(r'\.[mh]$')
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Preproc':
+            self._process_preproc(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_preproc(self, token, content):
+        if self.state:
+            self._process_import(token, content)
+
+        self.state = content
+
+    def _process_import(self, token, content):
+        if self.state == '#' and content.startswith('import '):
+            self.append(self._format(content))
+        self.state = None
+
+    def _process_other(self, token, content):
+        pass
+
+    def _format(self, content):
+        content = content.strip().lstrip('import ').strip()
+        content = content.strip('"').strip("'").strip()
+        content = content.strip('<').strip('>').strip()
+        content = content.split('/')[0]
+        content = self.extension.sub('', content, count=1)
+        return content

packages/wakatime/dependencies/php.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
-    wakatime.languages.php
-    ~~~~~~~~~~~~~~~~~~~~~~
+    wakatime.dependencies.php
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Parse dependencies from PHP code.
 
@@ -16,6 +16,10 @@ from ..compat import u
 class PhpParser(TokenParser):
     state = None
     parens = 0
+    exclude = [
+        r'^app$',
+        r'app\.php$',
+    ]
 
     def parse(self):
         for index, token, content in self.tokens:

packages/wakatime/dependencies/python.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
-    wakatime.languages.python
-    ~~~~~~~~~~~~~~~~~~~~~~~~~
+    wakatime.dependencies.python
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Parse dependencies from Python code.
 

packages/wakatime/dependencies/rust.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.rust
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Rust code.
+
+    :copyright: (c) 2018 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+
+
+class RustParser(TokenParser):
+    state = None
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Keyword':
+            self._process_keyword(token, content)
+        elif self.partial(token) == 'Whitespace':
+            self._process_whitespace(token, content)
+        elif self.partial(token) == 'Name':
+            self._process_name(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_keyword(self, token, content):
+        if self.state == 'extern' and content == 'crate':
+            self.state = 'extern crate'
+        else:
+            self.state = content
+
+    def _process_whitespace(self, token, content):
+        pass
+
+    def _process_name(self, token, content):
+        if self.state == 'extern crate':
+            self.append(content)
+        self.state = None
+
+    def _process_other(self, token, content):
+        self.state = None

packages/wakatime/dependencies/unknown.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
-    wakatime.languages.unknown
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+    wakatime.dependencies.unknown
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Parse dependencies from files of unknown language.
 

packages/wakatime/exceptions.py

@@ -12,3 +12,8 @@
 
 class NotYetImplemented(Exception):
     """This method needs to be implemented."""
+
+
+class SkipHeartbeat(Exception):
+    """Raised to prevent the current heartbeat from being sent."""
+    pass

packages/wakatime/heartbeat.py

@@ -10,11 +10,13 @@
 import os
 import logging
 import re
+from subprocess import PIPE
 
-from .compat import u, json
+from .compat import u, json, is_win, Popen
+from .exceptions import SkipHeartbeat
 from .project import get_project_info
 from .stats import get_file_stats
-from .utils import get_user_agent, should_exclude, format_file_path
+from .utils import get_user_agent, should_exclude, format_file_path, find_project_file
 
 
 log = logging.getLogger('WakaTime')
@@ -30,6 +32,7 @@ class Heartbeat(object):
     time = None
     entity = None
     type = None
+    category = None
     is_write = None
     project = None
     branch = None
@@ -40,7 +43,13 @@ class Heartbeat(object):
     cursorpos = None
     user_agent = None
 
+    _sensitive = ('dependencies', 'lines', 'lineno', 'cursorpos', 'branch')
+
     def __init__(self, data, args, configs, _clone=None):
+        if not data:
+            self.skip = u('Skipping because heartbeat data is missing.')
+            return
+
         self.args = args
         self.configs = configs
 
@@ -53,6 +62,21 @@ class Heartbeat(object):
         if self.type not in ['file', 'domain', 'app']:
             self.type = 'file'
 
+        self.category = data.get('category')
+        allowed_categories = [
+            'coding',
+            'building',
+            'indexing',
+            'debugging',
+            'running tests',
+            'manual testing',
+            'browsing',
+            'code reviewing',
+            'designing',
+        ]
+        if self.category not in allowed_categories:
+            self.category = None
+
         if not _clone:
             exclude = self._excluded_by_pattern()
             if exclude:
@@ -62,20 +86,37 @@ class Heartbeat(object):
                 return
             if self.type == 'file':
                 self.entity = format_file_path(self.entity)
-            if self.type == 'file' and not os.path.isfile(self.entity):
-                self.skip = u('File does not exist; ignoring this heartbeat.')
-                return
+                self._format_local_file()
+                if not self._file_exists():
+                    self.skip = u('File does not exist; ignoring this heartbeat.')
+                    return
+                if self._excluded_by_missing_project_file():
+                    self.skip = u('Skipping because missing .wakatime-project file in parent path.')
+                    return
+
+            if args.local_file and not os.path.isfile(args.local_file):
+                args.local_file = None
 
             project, branch = get_project_info(configs, self, data)
             self.project = project
             self.branch = branch
 
-            stats = get_file_stats(self.entity,
-                                   entity_type=self.type,
-                                   lineno=data.get('lineno'),
-                                   cursorpos=data.get('cursorpos'),
-                                   plugin=args.plugin,
-                                   language=data.get('language'))
+            if self._excluded_by_unknown_project():
+                self.skip = u('Skipping because project unknown.')
+                return
+
+            try:
+                stats = get_file_stats(self.entity,
+                                       entity_type=self.type,
+                                       lineno=data.get('lineno'),
+                                       cursorpos=data.get('cursorpos'),
+                                       plugin=args.plugin,
+                                       language=data.get('language'),
+                                       local_file=args.local_file)
+            except SkipHeartbeat as ex:
+                self.skip = u(ex) or 'Skipping'
+                return
+
         else:
             self.project = data.get('project')
             self.branch = data.get('branch')
@@ -91,7 +132,6 @@ class Heartbeat(object):
         data = self.dict()
         data.update(attrs)
         heartbeat = Heartbeat(data, self.args, self.configs, _clone=True)
-        heartbeat.skip = self.skip
         return heartbeat
 
     def sanitize(self):
@@ -100,7 +140,7 @@ class Heartbeat(object):
         Returns a Heartbeat.
         """
 
-        if not self.args.hidefilenames:
+        if not self.args.hide_file_names:
             return self
 
         if self.entity is None:
@@ -109,29 +149,12 @@ class Heartbeat(object):
         if self.type != 'file':
             return self
 
-        for pattern in self.args.hidefilenames:
-            try:
-                compiled = re.compile(pattern, re.IGNORECASE)
-                if compiled.search(self.entity):
-
-                    sanitized = {}
-                    sensitive = ['dependencies', 'lines', 'lineno', 'cursorpos', 'branch']
-                    for key, val in self.items():
-                        if key in sensitive:
-                            sanitized[key] = None
-                        else:
-                            sanitized[key] = val
-
-                    extension = u(os.path.splitext(self.entity)[1])
-                    sanitized['entity'] = u('HIDDEN{0}').format(extension)
-
-                    return self.update(sanitized)
-
-            except re.error as ex:
-                log.warning(u('Regex error ({msg}) for include pattern: {pattern}').format(
-                    msg=u(ex),
-                    pattern=u(pattern),
-                ))
+        if self.should_obfuscate_filename():
+            self._sanitize_metadata()
+            extension = u(os.path.splitext(self.entity)[1])
+            self.entity = u('HIDDEN{0}').format(extension)
+        elif self.should_obfuscate_project():
+            self._sanitize_metadata()
 
         return self
 
@@ -141,30 +164,175 @@ class Heartbeat(object):
     def dict(self):
         return {
             'time': self.time,
-            'entity': self.entity,
+            'entity': self._unicode(self.entity),
             'type': self.type,
+            'category': self.category,
             'is_write': self.is_write,
-            'project': self.project,
-            'branch': self.branch,
-            'language': self.language,
-            'dependencies': self.dependencies,
+            'project': self._unicode(self.project),
+            'branch': self._unicode(self.branch),
+            'language': self._unicode(self.language),
+            'dependencies': self._unicode_list(self.dependencies),
             'lines': self.lines,
             'lineno': self.lineno,
             'cursorpos': self.cursorpos,
-            'user_agent': self.user_agent,
+            'user_agent': self._unicode(self.user_agent),
         }
 
     def items(self):
         return self.dict().items()
 
     def get_id(self):
-        return u('{h.time}-{h.type}-{h.project}-{h.branch}-{h.entity}-{h.is_write}').format(
-            h=self,
+        return u('{time}-{type}-{category}-{project}-{branch}-{entity}-{is_write}').format(
+            time=self.time,
+            type=self.type,
+            category=self.category,
+            project=self._unicode(self.project),
+            branch=self._unicode(self.branch),
+            entity=self._unicode(self.entity),
+            is_write=self.is_write,
         )
 
+    def should_obfuscate_filename(self):
+        """Returns True if hide_file_names is true or the entity file path
+        matches one in the list of obfuscated file paths."""
+
+        for pattern in self.args.hide_file_names:
+            try:
+                compiled = re.compile(pattern, re.IGNORECASE)
+                if compiled.search(self.entity):
+                    return True
+            except re.error as ex:
+                log.warning(u('Regex error ({msg}) for hide_file_names pattern: {pattern}').format(
+                    msg=u(ex),
+                    pattern=u(pattern),
+                ))
+        return False
+
+    def should_obfuscate_project(self):
+        """Returns True if hide_project_names is true or the entity file path
+        matches one in the list of obfuscated project paths."""
+
+        for pattern in self.args.hide_project_names:
+            try:
+                compiled = re.compile(pattern, re.IGNORECASE)
+                if compiled.search(self.entity):
+                    return True
+            except re.error as ex:
+                log.warning(u('Regex error ({msg}) for hide_project_names pattern: {pattern}').format(
+                    msg=u(ex),
+                    pattern=u(pattern),
+                ))
+        return False
+
+    def _unicode(self, value):
+        if value is None:
+            return None
+        return u(value)
+
+    def _unicode_list(self, values):
+        if values is None:
+            return None
+        return [self._unicode(value) for value in values]
+
+    def _file_exists(self):
+        return (self.entity and os.path.isfile(self.entity) or
+            self.args.local_file and os.path.isfile(self.args.local_file))
+
+    def _format_local_file(self):
+        """When args.local_file empty on Windows, tries to map args.entity to a
+        unc path.
+
+        Updates args.local_file in-place without returning anything.
+        """
+
+        if self.type != 'file':
+            return
+
+        if not is_win:
+            return
+
+        if self._file_exists():
+            return
+
+        self.args.local_file = self._to_unc_path(self.entity)
+
+    def _to_unc_path(self, filepath):
+        drive, rest = self._splitdrive(filepath)
+        if not drive:
+            return filepath
+
+        stdout = None
+        try:
+            stdout, stderr = Popen(['net', 'use'], stdout=PIPE, stderr=PIPE).communicate()
+        except OSError:
+            pass
+        else:
+            if stdout:
+                cols = None
+                for line in stdout.strip().splitlines()[1:]:
+                    line = u(line)
+                    if not line.strip():
+                        continue
+                    if not cols:
+                        cols = self._unc_columns(line)
+                        continue
+                    start, end = cols.get('local', (0, 0))
+                    if not start and not end:
+                        break
+                    local = line[start:end].strip().split(':')[0].upper()
+                    if not local.isalpha():
+                        continue
+                    if local == drive:
+                        start, end = cols.get('remote', (0, 0))
+                        if not start and not end:
+                            break
+                        remote = line[start:end].strip()
+                        return remote + rest
+
+        return filepath
+
+    def _unc_columns(self, line):
+        cols = {}
+        current_col = u('')
+        newcol = False
+        start, end = 0, 0
+        for char in line:
+            if char.isalpha():
+                if newcol:
+                    cols[current_col.strip().lower()] = (start, end)
+                    current_col = u('')
+                    start = end
+                    newcol = False
+                current_col += u(char)
+            else:
+                newcol = True
+            end += 1
+        if start != end and current_col:
+            cols[current_col.strip().lower()] = (start, -1)
+        return cols
+
+    def _splitdrive(self, filepath):
+        if filepath[1:2] != ':' or not filepath[0].isalpha():
+            return None, filepath
+        return filepath[0].upper(), filepath[2:]
+
     def _excluded_by_pattern(self):
         return should_exclude(self.entity, self.args.include, self.args.exclude)
 
+    def _excluded_by_unknown_project(self):
+        if self.project:
+            return False
+        return self.args.exclude_unknown_project
+
+    def _excluded_by_missing_project_file(self):
+        if not self.args.include_only_with_project_file:
+            return False
+        return find_project_file(self.entity) is None
+
+    def _sanitize_metadata(self):
+        for key in self._sensitive:
+            setattr(self, key, None)
+
     def __repr__(self):
         return self.json()
 

packages/wakatime/languages/default.json

@@ -1,81 +1,81 @@
 {
-  "actionscript": "ActionScript", 
-  "apacheconf": "ApacheConf", 
-  "applescript": "AppleScript", 
-  "asp": "ASP", 
-  "assembly": "Assembly", 
-  "awk": "Awk", 
-  "bash": "Bash", 
-  "basic": "Basic", 
-  "brightscript": "BrightScript", 
-  "c": "C", 
-  "c#": "C#", 
-  "c++": "C++", 
-  "clojure": "Clojure", 
-  "cocoa": "Cocoa", 
-  "coffeescript": "CoffeeScript", 
-  "coldfusion": "ColdFusion", 
-  "common lisp": "Common Lisp", 
-  "cshtml": "CSHTML", 
-  "css": "CSS", 
-  "dart": "Dart", 
-  "delphi": "Delphi", 
-  "elixir": "Elixir", 
-  "elm": "Elm", 
-  "emacs lisp": "Emacs Lisp", 
-  "erlang": "Erlang", 
-  "f#": "F#", 
-  "fortran": "Fortran", 
-  "go": "Go", 
-  "gous": "Gosu", 
-  "groovy": "Groovy", 
-  "haml": "Haml", 
-  "haskell": "Haskell", 
-  "haxe": "Haxe", 
-  "html": "HTML", 
-  "ini": "INI", 
-  "jade": "Jade", 
-  "java": "Java", 
-  "javascript": "JavaScript", 
-  "json": "JSON", 
-  "jsx": "JSX", 
-  "kotlin": "Kotlin", 
-  "less": "LESS", 
-  "lua": "Lua", 
-  "markdown": "Markdown", 
-  "matlab": "Matlab", 
-  "mustache": "Mustache", 
-  "objective-c": "Objective-C", 
-  "objective-c++": "Objective-C++", 
-  "objective-j": "Objective-J", 
-  "ocaml": "OCaml", 
-  "perl": "Perl", 
-  "php": "PHP", 
-  "powershell": "PowerShell", 
-  "prolog": "Prolog", 
-  "puppet": "Puppet", 
-  "python": "Python", 
-  "r": "R", 
-  "restructuredtext": "reStructuredText", 
-  "ruby": "Ruby", 
-  "rust": "Rust", 
-  "sass": "Sass", 
-  "scala": "Scala", 
-  "scheme": "Scheme", 
-  "scss": "SCSS", 
-  "shell": "Shell", 
-  "slim": "Slim", 
-  "smalltalk": "Smalltalk", 
-  "sql": "SQL", 
-  "swift": "Swift", 
-  "text": "Text", 
-  "turing": "Turing", 
-  "twig": "Twig", 
-  "typescript": "TypeScript", 
-  "typoscript": "TypoScript", 
-  "vb.net": "VB.net", 
-  "viml": "VimL", 
-  "xaml": "XAML", 
-  "xml": "XML", 
+  "actionscript": "ActionScript",
+  "apacheconf": "ApacheConf",
+  "applescript": "AppleScript",
+  "asp": "ASP",
+  "assembly": "Assembly",
+  "awk": "Awk",
+  "bash": "Bash",
+  "basic": "Basic",
+  "brightscript": "BrightScript",
+  "c": "C",
+  "c#": "C#",
+  "c++": "C++",
+  "clojure": "Clojure",
+  "cocoa": "Cocoa",
+  "coffeescript": "CoffeeScript",
+  "coldfusion": "ColdFusion",
+  "common lisp": "Common Lisp",
+  "cshtml": "CSHTML",
+  "css": "CSS",
+  "dart": "Dart",
+  "delphi": "Delphi",
+  "elixir": "Elixir",
+  "elm": "Elm",
+  "emacs lisp": "Emacs Lisp",
+  "erlang": "Erlang",
+  "f#": "F#",
+  "fortran": "Fortran",
+  "go": "Go",
+  "gosu": "Gosu",
+  "groovy": "Groovy",
+  "haml": "Haml",
+  "haskell": "Haskell",
+  "haxe": "Haxe",
+  "html": "HTML",
+  "ini": "INI",
+  "jade": "Jade",
+  "java": "Java",
+  "javascript": "JavaScript",
+  "json": "JSON",
+  "jsx": "JSX",
+  "kotlin": "Kotlin",
+  "less": "LESS",
+  "lua": "Lua",
+  "markdown": "Markdown",
+  "matlab": "Matlab",
+  "mustache": "Mustache",
+  "objective-c": "Objective-C",
+  "objective-c++": "Objective-C++",
+  "objective-j": "Objective-J",
+  "ocaml": "OCaml",
+  "perl": "Perl",
+  "php": "PHP",
+  "powershell": "PowerShell",
+  "prolog": "Prolog",
+  "puppet": "Puppet",
+  "python": "Python",
+  "r": "R",
+  "restructuredtext": "reStructuredText",
+  "ruby": "Ruby",
+  "rust": "Rust",
+  "sass": "Sass",
+  "scala": "Scala",
+  "scheme": "Scheme",
+  "scss": "SCSS",
+  "shell": "Shell",
+  "slim": "Slim",
+  "smalltalk": "Smalltalk",
+  "sql": "SQL",
+  "swift": "Swift",
+  "text": "Text",
+  "turing": "Turing",
+  "twig": "Twig",
+  "typescript": "TypeScript",
+  "typoscript": "TypoScript",
+  "vb.net": "VB.net",
+  "viml": "VimL",
+  "xaml": "XAML",
+  "xml": "XML",
   "yaml": "YAML"
 }

packages/wakatime/logger.py

@@ -75,7 +75,7 @@ def setup_logging(args, version):
     for handler in logger.handlers:
         logger.removeHandler(handler)
     set_log_level(logger, args)
-    logfile = args.logfile
+    logfile = args.log_file
     if not logfile:
         logfile = '~/.wakatime.log'
     handler = logging.FileHandler(os.path.expanduser(logfile))

packages/wakatime/main.py

@@ -14,6 +14,7 @@ from __future__ import print_function
 import logging
 import os
 import sys
+import time
 import traceback
 
 pwd = os.path.dirname(os.path.abspath(__file__))
@@ -24,7 +25,7 @@ from .__about__ import __version__
 from .api import send_heartbeats
 from .arguments import parse_arguments
 from .compat import u, json
-from .constants import SUCCESS, UNKNOWN_ERROR
+from .constants import SUCCESS, UNKNOWN_ERROR, HEARTBEATS_PER_REQUEST
 from .logger import setup_logging
 
 log = logging.getLogger('WakaTime')
@@ -63,12 +64,23 @@ def execute(argv=None):
                     msg=u(ex),
                 ))
 
-        retval = send_heartbeats(heartbeats, args, configs)
+        retval = SUCCESS
+        while heartbeats:
+            retval = send_heartbeats(heartbeats[:HEARTBEATS_PER_REQUEST], args, configs)
+            heartbeats = heartbeats[HEARTBEATS_PER_REQUEST:]
+            if retval != SUCCESS:
+                break
+
+        if heartbeats:
+            Queue(args, configs).push_many(heartbeats)
+
         if retval == SUCCESS:
             queue = Queue(args, configs)
-            offline_heartbeats = queue.pop_many()
-            if len(offline_heartbeats) > 0:
+            for offline_heartbeats in queue.pop_many(args.sync_offline_activity):
+                time.sleep(1)
                 retval = send_heartbeats(offline_heartbeats, args, configs)
+                if retval != SUCCESS:
+                    break
 
         return retval
 

packages/wakatime/offlinequeue.py

@@ -15,6 +15,7 @@ import os
 from time import sleep
 
 from .compat import json
+from .constants import DEFAULT_SYNC_OFFLINE_ACTIVITY, HEARTBEATS_PER_REQUEST
 from .heartbeat import Heartbeat
 
 
@@ -104,19 +105,23 @@ class Queue(object):
 
     def pop_many(self, limit=None):
         if limit is None:
-            limit = 100
+            limit = DEFAULT_SYNC_OFFLINE_ACTIVITY
 
         heartbeats = []
 
         count = 0
-        while limit == 0 or count < limit:
+        while count < limit:
             heartbeat = self.pop()
             if not heartbeat:
                 break
             heartbeats.append(heartbeat)
             count += 1
+            if count % HEARTBEATS_PER_REQUEST == 0:
+                yield heartbeats
+                heartbeats = []
 
-        return heartbeats
+        if heartbeats:
+            yield heartbeats
 
     def _get_db_file(self):
         home = '~'

packages/wakatime/packages/pygments/lexers/javascript.py

@@ -37,7 +37,7 @@ class JavascriptLexer(RegexLexer):
 
     name = 'JavaScript'
     aliases = ['js', 'javascript']
-    filenames = ['*.js', '*.jsm']
+    filenames = ['*.js', '*.jsm', '*.mjs']
     mimetypes = ['application/javascript', 'application/x-javascript',
                  'text/x-javascript', 'text/javascript']
 
@@ -1035,7 +1035,6 @@ class CoffeeScriptLexer(RegexLexer):
     filenames = ['*.coffee']
     mimetypes = ['text/coffeescript']
 
-
     _operator_re = (
         r'\+\+|~|&&|\band\b|\bor\b|\bis\b|\bisnt\b|\bnot\b|\?|:|'
         r'\|\||\\(?=\n)|'
@@ -1464,6 +1463,7 @@ class EarlGreyLexer(RegexLexer):
         ],
     }
 
+
 class JuttleLexer(RegexLexer):
     """
     For `Juttle`_ source code.

packages/wakatime/project.py

@@ -9,8 +9,11 @@
     :license: BSD, see LICENSE for more details.
 """
 
+import os
 import logging
+import random
 
+from .compat import open
 from .projects.git import Git
 from .projects.mercurial import Mercurial
 from .projects.projectfile import ProjectFile
@@ -65,6 +68,8 @@ def get_project_info(configs, heartbeat, data):
     if project_name is None:
         project_name = data.get('project') or heartbeat.args.project
 
+    hide_project = heartbeat.should_obfuscate_project()
+
     if project_name is None or branch_name is None:
 
         for plugin_cls in REV_CONTROL_PLUGINS:
@@ -76,9 +81,18 @@ def get_project_info(configs, heartbeat, data):
             if project.process():
                 project_name = project_name or project.name()
                 branch_name = branch_name or project.branch()
+                if hide_project:
+                    branch_name = None
+                    project_name = generate_project_name()
+                    project_file = os.path.join(project.folder(), '.wakatime-project')
+                    try:
+                        with open(project_file, 'w') as fh:
+                            fh.write(project_name)
+                    except IOError:
+                        project_name = None
                 break
 
-    if project_name is None:
+    if project_name is None and not hide_project:
         project_name = data.get('alternate_project') or heartbeat.args.alternate_project
 
     return project_name, branch_name
@@ -88,3 +102,42 @@ def get_configs_for_plugin(plugin_name, configs):
     if configs and configs.has_section(plugin_name):
         return dict(configs.items(plugin_name))
     return None
+
+
+def generate_project_name():
+    """Generates a random project name."""
+
+    adjectives = [
+        'aged', 'ancient', 'autumn', 'billowing', 'bitter', 'black', 'blue', 'bold',
+        'broad', 'broken', 'calm', 'cold', 'cool', 'crimson', 'curly', 'damp',
+        'dark', 'dawn', 'delicate', 'divine', 'dry', 'empty', 'falling', 'fancy',
+        'flat', 'floral', 'fragrant', 'frosty', 'gentle', 'green', 'hidden', 'holy',
+        'icy', 'jolly', 'late', 'lingering', 'little', 'lively', 'long', 'lucky',
+        'misty', 'morning', 'muddy', 'mute', 'nameless', 'noisy', 'odd', 'old',
+        'orange', 'patient', 'plain', 'polished', 'proud', 'purple', 'quiet', 'rapid',
+        'raspy', 'red', 'restless', 'rough', 'round', 'royal', 'shiny', 'shrill',
+        'shy', 'silent', 'small', 'snowy', 'soft', 'solitary', 'sparkling', 'spring',
+        'square', 'steep', 'still', 'summer', 'super', 'sweet', 'throbbing', 'tight',
+        'tiny', 'twilight', 'wandering', 'weathered', 'white', 'wild', 'winter', 'wispy',
+        'withered', 'yellow', 'young'
+    ]
+    nouns = [
+        'art', 'band', 'bar', 'base', 'bird', 'block', 'boat', 'bonus',
+        'bread', 'breeze', 'brook', 'bush', 'butterfly', 'cake', 'cell', 'cherry',
+        'cloud', 'credit', 'darkness', 'dawn', 'dew', 'disk', 'dream', 'dust',
+        'feather', 'field', 'fire', 'firefly', 'flower', 'fog', 'forest', 'frog',
+        'frost', 'glade', 'glitter', 'grass', 'hall', 'hat', 'haze', 'heart',
+        'hill', 'king', 'lab', 'lake', 'leaf', 'limit', 'math', 'meadow',
+        'mode', 'moon', 'morning', 'mountain', 'mouse', 'mud', 'night', 'paper',
+        'pine', 'poetry', 'pond', 'queen', 'rain', 'recipe', 'resonance', 'rice',
+        'river', 'salad', 'scene', 'sea', 'shadow', 'shape', 'silence', 'sky',
+        'smoke', 'snow', 'snowflake', 'sound', 'star', 'sun', 'sun', 'sunset',
+        'surf', 'term', 'thunder', 'tooth', 'tree', 'truth', 'union', 'unit',
+        'violet', 'voice', 'water', 'waterfall', 'wave', 'wildflower', 'wind', 'wood'
+    ]
+    numbers = [str(x) for x in range(10)]
+    return ' '.join([
+        random.choice(adjectives).capitalize(),
+        random.choice(nouns).capitalize(),
+        random.choice(numbers) + random.choice(numbers),
+    ])

packages/wakatime/projects/base.py

@@ -43,3 +43,8 @@ class BaseProject(object):
         """ Returns the current branch.
         """
         raise NotYetImplemented()
+
+    def folder(self):
+        """ Returns the project's top folder path.
+        """
+        raise NotYetImplemented()

packages/wakatime/projects/git.py

@@ -25,6 +25,7 @@ class Git(BaseProject):
     _submodule = None
     _project_name = None
     _head_file = None
+    _project_folder = None
 
     def process(self):
         return self._find_git_config_file(self.path)
@@ -35,19 +36,14 @@ class Git(BaseProject):
     def branch(self):
         head = self._head_file
         if head:
-            try:
-                with open(head, 'r', encoding='utf-8') as fh:
-                    return self._get_branch_from_head_file(fh.readline())
-            except UnicodeDecodeError:  # pragma: nocover
-                try:
-                    with open(head, 'r', encoding=sys.getfilesystemencoding()) as fh:
-                        return self._get_branch_from_head_file(fh.readline())
-                except:
-                    log.traceback(logging.WARNING)
-            except IOError:  # pragma: nocover
-                log.traceback(logging.WARNING)
+            line = self._first_line_of_file(head)
+            if line is not None:
+                return self._get_branch_from_head_file(line)
         return u('master')
 
+    def folder(self):
+        return self._project_folder
+
     def _find_git_config_file(self, path):
         path = os.path.realpath(path)
         if os.path.isfile(path):
@@ -55,13 +51,26 @@ class Git(BaseProject):
         if os.path.isfile(os.path.join(path, '.git', 'config')):
             self._project_name = os.path.basename(path)
             self._head_file = os.path.join(path, '.git', 'HEAD')
+            self._project_folder = path
             return True
-        if self._submodules_supported_for_path(path):
-            submodule_path = self._find_path_from_submodule(path)
-            if submodule_path:
-                self._project_name = os.path.basename(path)
-                self._head_file = os.path.join(submodule_path, 'HEAD')
+
+        link_path = self._path_from_gitdir_link_file(path)
+        if link_path:
+
+            # first check if this is a worktree
+            if self._is_worktree(link_path):
+                self._project_name = self._project_from_worktree(link_path)
+                self._head_file = os.path.join(link_path, 'HEAD')
+                self._project_folder = path
                 return True
+
+            # next check if this is a submodule
+            if self._submodules_supported_for_path(path):
+                self._project_name = os.path.basename(path)
+                self._head_file = os.path.join(link_path, 'HEAD')
+                self._project_folder = path
+                return True
+
         split_path = os.path.split(path)
         if split_path[1] == '':
             return False
@@ -77,8 +86,6 @@ class Git(BaseProject):
             return True
 
         disabled = self._configs.get('submodules_disabled')
-        if not disabled:
-            return True
 
         if disabled.strip().lower() == 'true':
             return False
@@ -99,30 +106,51 @@ class Git(BaseProject):
 
         return True
 
-    def _find_path_from_submodule(self, path):
+    def _is_worktree(self, link_path):
+        return os.path.basename(os.path.dirname(link_path)) == 'worktrees'
+
+    def _path_from_gitdir_link_file(self, path):
         link = os.path.join(path, '.git')
         if not os.path.isfile(link):
             return None
 
+        line = self._first_line_of_file(link)
+        if line is not None:
+            return self._path_from_gitdir_string(path, line)
+
+        return None
+
+    def _path_from_gitdir_string(self, path, line):
+        if line.startswith('gitdir: '):
+            subpath = line[len('gitdir: '):].strip()
+            if os.path.isfile(os.path.join(path, subpath, 'HEAD')):
+                return os.path.realpath(os.path.join(path, subpath))
+
+        return None
+
+    def _project_from_worktree(self, link_path):
+        commondir = os.path.join(link_path, 'commondir')
+        if os.path.isfile(commondir):
+            line = self._first_line_of_file(commondir)
+            if line:
+                gitdir = os.path.abspath(os.path.join(link_path, line))
+                if os.path.basename(gitdir) == '.git':
+                    return os.path.basename(os.path.dirname(gitdir))
+
+        return None
+
+    def _first_line_of_file(self, filepath):
         try:
-            with open(link, 'r', encoding='utf-8') as fh:
-                return self._get_path_from_submodule_link(path, fh.readline())
+            with open(filepath, 'r', encoding='utf-8') as fh:
+                return fh.readline().strip()
         except UnicodeDecodeError:
-            try:
-                with open(link, 'r', encoding=sys.getfilesystemencoding()) as fh:
-                    return self._get_path_from_submodule_link(path, fh.readline())
-            except:
-                log.traceback(logging.WARNING)
+            pass
         except IOError:
+            pass
+        try:
+            with open(filepath, 'r', encoding=sys.getfilesystemencoding()) as fh:
+                return fh.readline().strip()
+        except:
             log.traceback(logging.WARNING)
 
         return None
-
-    def _get_path_from_submodule_link(self, path, line):
-        if line.startswith('gitdir: '):
-            subpath = line[len('gitdir: '):].strip()
-            if os.path.isfile(os.path.join(path, subpath, 'config')) and \
-                    os.path.isfile(os.path.join(path, subpath, 'HEAD')):
-                return os.path.join(path, subpath)
-
-        return None

packages/wakatime/projects/mercurial.py

@@ -47,6 +47,11 @@ class Mercurial(BaseProject):
                 log.traceback(logging.WARNING)
         return u('default')
 
+    def folder(self):
+        if self.configDir:
+            return os.path.dirname(self.configDir)
+        return None
+
     def _find_hg_config_dir(self, path):
         path = os.path.realpath(path)
         if os.path.isfile(path):

packages/wakatime/projects/projectfile.py

@@ -12,11 +12,11 @@
 """
 
 import logging
-import os
 import sys
 
 from .base import BaseProject
 from ..compat import u, open
+from ..utils import find_project_file
 
 
 log = logging.getLogger('WakaTime')
@@ -25,7 +25,7 @@ log = logging.getLogger('WakaTime')
 class ProjectFile(BaseProject):
 
     def process(self):
-        self.config = self._find_config(self.path)
+        self.config = find_project_file(self.path)
         self._project_name = None
         self._project_branch = None
 
@@ -33,13 +33,13 @@ class ProjectFile(BaseProject):
 
             try:
                 with open(self.config, 'r', encoding='utf-8') as fh:
-                    self._project_name = u(fh.readline().strip())
-                    self._project_branch = u(fh.readline().strip())
+                    self._project_name = u(fh.readline().strip()) or None
+                    self._project_branch = u(fh.readline().strip()) or None
             except UnicodeDecodeError:  # pragma: nocover
                 try:
                     with open(self.config, 'r', encoding=sys.getfilesystemencoding()) as fh:
-                        self._project_name = u(fh.readline().strip())
-                        self._project_branch = u(fh.readline().strip())
+                        self._project_name = u(fh.readline().strip()) or None
+                        self._project_branch = u(fh.readline().strip()) or None
                 except:
                     log.traceback(logging.WARNING)
             except IOError:  # pragma: nocover
@@ -53,14 +53,3 @@ class ProjectFile(BaseProject):
 
     def branch(self):
         return self._project_branch
-
-    def _find_config(self, path):
-        path = os.path.realpath(path)
-        if os.path.isfile(path):
-            path = os.path.split(path)[0]
-        if os.path.isfile(os.path.join(path, '.wakatime-project')):
-            return os.path.join(path, '.wakatime-project')
-        split_path = os.path.split(path)
-        if split_path[1] == '':
-            return None
-        return self._find_config(split_path[0])

packages/wakatime/projects/subversion.py

@@ -12,10 +12,10 @@
 import logging
 import os
 import platform
-from subprocess import Popen, PIPE
+from subprocess import PIPE
 
 from .base import BaseProject
-from ..compat import u, open
+from ..compat import u, open, Popen
 try:
     from collections import OrderedDict
 except ImportError:  # pragma: nocover
@@ -41,6 +41,11 @@ class Subversion(BaseProject):
             return None  # pragma: nocover
         return u(self.info['URL'].split('/')[-1].split('\\')[-1])
 
+    def folder(self):
+        if 'Repository Root' not in self.info:
+            return None
+        return self.info['Repository Root']
+
     def _find_binary(self):
         if self.binary_location:
             return self.binary_location
@@ -65,24 +70,22 @@ class Subversion(BaseProject):
         if not self._is_mac() or self._has_xcode_tools():
             stdout = None
             try:
-                os.environ['LANG'] = 'en_US'
-                stdout, stderr = Popen([
-                    self._find_binary(), 'info', os.path.realpath(path)
-                ], stdout=PIPE, stderr=PIPE).communicate()
+                stdout, stderr = Popen(
+                    [self._find_binary(), 'info', os.path.realpath(path)],
+                    stdout=PIPE,
+                    stderr=PIPE,
+                ).communicate()
             except OSError:
                 pass
             else:
                 if stdout:
                     for line in stdout.splitlines():
-                        line = u(line)
-                        line = line.split(': ', 1)
+                        line = u(line).split(': ', 1)
                         if len(line) == 2:
                             info[line[0]] = line[1]
         return info
 
     def _find_project_base(self, path, found=False):
-        if platform.system() == 'Windows':
-            return False  # pragma: nocover
         path = os.path.realpath(path)
         if os.path.isfile(path):
             path = os.path.split(path)[0]

packages/wakatime/stats.py

@@ -17,6 +17,7 @@ import sys
 from .compat import u, open
 from .constants import MAX_FILE_SIZE_SUPPORTED
 from .dependencies import DependencyParser
+from .exceptions import SkipHeartbeat
 from .language_priorities import LANGUAGES
 
 from .packages.pygments.lexers import (
@@ -39,7 +40,7 @@ log = logging.getLogger('WakaTime')
 
 
 def get_file_stats(file_name, entity_type='file', lineno=None, cursorpos=None,
-                   plugin=None, language=None):
+                   plugin=None, language=None, local_file=None):
     if entity_type != 'file':
         stats = {
             'language': None,
@@ -51,22 +52,24 @@ def get_file_stats(file_name, entity_type='file', lineno=None, cursorpos=None,
     else:
         language, lexer = standardize_language(language, plugin)
         if not language:
-            language, lexer = guess_language(file_name)
+            language, lexer = guess_language(file_name, local_file)
 
-        parser = DependencyParser(file_name, lexer)
+        language = use_root_language(language, lexer)
+
+        parser = DependencyParser(local_file or file_name, lexer)
         dependencies = parser.parse()
 
         stats = {
             'language': language,
             'dependencies': dependencies,
-            'lines': number_lines_in_file(file_name),
+            'lines': number_lines_in_file(local_file or file_name),
             'lineno': lineno,
             'cursorpos': cursorpos,
         }
     return stats
 
 
-def guess_language(file_name):
+def guess_language(file_name, local_file):
     """Guess lexer and language for a file.
 
     Returns a tuple of (language_str, lexer_obj).
@@ -78,14 +81,14 @@ def guess_language(file_name):
     if language:
         lexer = get_lexer(language)
     else:
-        lexer = smart_guess_lexer(file_name)
+        lexer = smart_guess_lexer(file_name, local_file)
         if lexer:
             language = u(lexer.name)
 
     return language, lexer
 
 
-def smart_guess_lexer(file_name):
+def smart_guess_lexer(file_name, local_file):
     """Guess Pygments lexer for a file.
 
     Looks for a vim modeline in file contents, then compares the accuracy
@@ -96,7 +99,7 @@ def smart_guess_lexer(file_name):
 
     text = get_file_head(file_name)
 
-    lexer1, accuracy1 = guess_lexer_using_filename(file_name, text)
+    lexer1, accuracy1 = guess_lexer_using_filename(local_file or file_name, text)
     lexer2, accuracy2 = guess_lexer_using_modeline(text)
 
     if lexer1:
@@ -118,6 +121,8 @@ def guess_lexer_using_filename(file_name, text):
 
     try:
         lexer = custom_pygments_guess_lexer_for_filename(file_name, text)
+    except SkipHeartbeat as ex:
+        raise SkipHeartbeat(u(ex))
     except:
         log.traceback(logging.DEBUG)
 
@@ -167,17 +172,29 @@ def get_language_from_extension(file_name):
 
     filepart, extension = os.path.splitext(file_name)
 
-    if re.match(r'\.h.*', extension, re.IGNORECASE) or re.match(r'\.c.*', extension, re.IGNORECASE):
+    if re.match(r'\.h.*$', extension, re.IGNORECASE) or re.match(r'\.c.*$', extension, re.IGNORECASE):
 
         if os.path.exists(u('{0}{1}').format(u(filepart), u('.c'))) or os.path.exists(u('{0}{1}').format(u(filepart), u('.C'))):
             return 'C'
 
+        if os.path.exists(u('{0}{1}').format(u(filepart), u('.m'))) or os.path.exists(u('{0}{1}').format(u(filepart), u('.M'))):
+            return 'Objective-C'
+
+        if os.path.exists(u('{0}{1}').format(u(filepart), u('.mm'))) or os.path.exists(u('{0}{1}').format(u(filepart), u('.MM'))):
+            return 'Objective-C++'
+
         available_extensions = extensions_in_same_folder(file_name)
         if '.cpp' in available_extensions:
             return 'C++'
         if '.c' in available_extensions:
             return 'C'
 
+    if re.match(r'\.m$', extension, re.IGNORECASE) and (os.path.exists(u('{0}{1}').format(u(filepart), u('.h'))) or os.path.exists(u('{0}{1}').format(u(filepart), u('.H')))):
+        return 'Objective-C'
+
+    if re.match(r'\.mm$', extension, re.IGNORECASE) and (os.path.exists(u('{0}{1}').format(u(filepart), u('.h'))) or os.path.exists(u('{0}{1}').format(u(filepart), u('.H')))):
+        return 'Objective-C++'
+
     return None
 
 
@@ -236,6 +253,13 @@ def get_lexer(language):
     return None
 
 
+def use_root_language(language, lexer):
+    if lexer and hasattr(lexer, 'root_lexer'):
+        return u(lexer.root_lexer.name)
+
+    return language
+
+
 def get_language_from_json(language, key):
     """Finds the given language in a json file."""
 
@@ -299,6 +323,12 @@ def custom_pygments_guess_lexer_for_filename(_fn, _text, **options):
             return lexer(**options)
         result.append(customize_lexer_priority(_fn, rv, lexer))
 
+    matlab = list(filter(lambda x: x[2].name.lower() == 'matlab', result))
+    if len(matlab) > 0:
+        objc = list(filter(lambda x: x[2].name.lower() == 'objective-c', result))
+        if objc and objc[0][0] == matlab[0][0]:
+            raise SkipHeartbeat('Skipping because not enough language accuracy.')
+
     def type_sort(t):
         # sort by:
         # - analyse score
@@ -322,7 +352,17 @@ def customize_lexer_priority(file_name, accuracy, lexer):
     elif lexer_name == 'matlab':
         available_extensions = extensions_in_same_folder(file_name)
         if '.mat' in available_extensions:
-            priority = 0.06
+            accuracy += 0.01
+        if '.h' not in available_extensions:
+            accuracy += 0.01
+    elif lexer_name == 'objective-c':
+        available_extensions = extensions_in_same_folder(file_name)
+        if '.mat' in available_extensions:
+            accuracy -= 0.01
+        else:
+            accuracy += 0.01
+        if '.h' in available_extensions:
+            accuracy += 0.01
 
     return (accuracy, priority, lexer)
 

packages/wakatime/utils.py

@@ -24,6 +24,10 @@ from .compat import u
 log = logging.getLogger('WakaTime')
 
 
+BACKSLASH_REPLACE_PATTERN = re.compile(r'[\\/]+')
+WINDOWS_DRIVE_PATTERN = re.compile(r'^[a-z]:/')
+
+
 def should_exclude(entity, include, exclude):
     if entity is not None and entity.strip() != '':
         for pattern in include:
@@ -74,11 +78,25 @@ def format_file_path(filepath):
 
     try:
         filepath = os.path.realpath(os.path.abspath(filepath))
-        filepath = re.sub(r'[/\\]', os.path.sep, filepath)
-    except:  # pragma: nocover
+        filepath = re.sub(BACKSLASH_REPLACE_PATTERN, '/', filepath)
+        if WINDOWS_DRIVE_PATTERN.match(filepath):
+            filepath = filepath.capitalize()
+    except:
         pass
     return filepath
 
 
 def get_hostname(args):
     return args.hostname or socket.gethostname()
+
+
+def find_project_file(path):
+    path = os.path.realpath(path)
+    if os.path.isfile(path):
+        path = os.path.split(path)[0]
+    if os.path.isfile(os.path.join(path, '.wakatime-project')):
+        return os.path.join(path, '.wakatime-project')
+    split_path = os.path.split(path)
+    if split_path[1] == '':
+        return None
+    return find_project_file(split_path[0])