v8.3.2

HISTORY.rst

@@ -3,12 +3,153 @@ History
 -------
 
 
+8.3.2 (2018-10-06)
+++++++++++++++++++
+
+- Send buffered heartbeats to API every 30 seconds.
+
+
+8.3.1 (2018-10-05)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.4.1.
+- Send 50 offline heartbeats to API per request with 1 second delay in between.
+
+
+8.3.0 (2018-10-03)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.4.0.
+- Support logging coding activity to remote network drive files on Windows
+  platform by detecting UNC path from drive letter.
+  `wakatime#72 <https://github.com/wakatime/wakatime/issues/72>`_
+
+
+8.2.0 (2018-09-30)
+++++++++++++++++++
+
+- Prevent opening cmd window on Windows when running wakatime-cli.
+  `#91 <https://github.com/wakatime/sublime-wakatime/issues/91>`_
+- Upgrade wakatime-cli to v10.3.0.
+- Re-enable detecting projects from Subversion folder on Windows platform.
+- Prevent opening cmd window on Windows when detecting project from Subversion.
+- Run tests on Windows using Appveyor.
+
+
+8.1.2 (2018-09-20)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.2.4.
+- Default --sync-offline-activity to 100 instead of 5, so offline coding is
+  synced to dashboard faster.
+- Batch heartbeats in groups of 10 per api request.
+- New config hide_project_name and argument --hide-project-names for
+  obfuscating project names when sending coding activity to api.
+- Fix mispelled Gosu language.
+  `wakatime#137 <https://github.com/wakatime/wakatime/issues/137>`_
+- Remove metadata when hiding project or file names.
+- New --local-file argument to be used when --entity is a remote file.
+- New argument --sync-offline-activity for configuring the maximum offline
+  heartbeats to sync to the WakaTime API.
+
+
+8.1.1 (2018-04-26)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.2.1.
+- Force forward slash for file paths.
+- New --category argument.
+- New --exclude-unknown-project argument and corresponding config setting.
+- Support for project detection from git worktree folders.
+
+
+8.1.0 (2018-04-03)
+++++++++++++++++++
+
+- Prefer Python3 over Python2 when running wakatime-cli core.
+- Improve detection of Python3 on Ubuntu 17.10 platforms.
+
+
+8.0.8 (2018-03-15)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.1.3.
+- Smarter C vs C++ vs Objective-C language detection.
+
+
+8.0.7 (2018-03-15)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.1.2.
+- Detect dependencies from Swift, Objective-C, TypeScript and JavaScript files.
+- Categorize .mjs files as JavaScript.
+  `#wakatime121 <https://github.com/wakatime/wakatime/issues/121>`_
+- Detect dependencies from Elm, Haskell, Haxe, Kotlin, Rust, and Scala files.
+- Improved Matlab vs Objective-C language detection.
+  `#wakatime129 <https://github.com/wakatime/wakatime/issues/129>`_
+
+
+8.0.6 (2018-01-04)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.1.0.
+- Ability to only track folders containing a .wakatime-project file using new
+  include_only_with_project_file argument and config option.
+
+
+8.0.5 (2017-11-24)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.0.5.
+- Fix bug that caused heartbeats to be cached locally instead of sent to API.
+
+
+8.0.4 (2017-11-23)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.0.4.
+- Improve Java dependency detection.
+- Skip null or missing heartbeats from extra heartbeats argument.
+
+
+8.0.3 (2017-11-22)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.0.3.
+- Support saving unicode heartbeats when working offline.
+  `wakatime#112 <https://github.com/wakatime/wakatime/issues/112>`_
+
+
+8.0.2 (2017-11-15)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.0.2.
+- Limit bulk syncing to 5 heartbeats per request.
+  `wakatime#109 <https://github.com/wakatime/wakatime/issues/109>`_
+
+
+8.0.1 (2017-11-09)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.0.1.
+- Parse array of results from bulk heartbeats endpoint, only saving heartbeats
+  to local offline cache when they were not accepted by the api.
+
+
+8.0.0 (2017-11-08)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.0.0.
+- Upload multiple heartbeats to bulk endpoint for improved network performance.
+  `wakatime#107 <https://github.com/wakatime/wakatime/issues/107>`_
+
+
 7.0.26 (2017-11-07)
 ++++++++++++++++++
 
 - Upgrade wakatime-cli to v9.0.1.
 - Fix bug causing 401 response when hidefilenames is enabled.
-  `#wakatime/106 <https://github.com/wakatime/wakatime/issues/106>`_
+  `wakatime#106 <https://github.com/wakatime/wakatime/issues/106>`_
 
 
 7.0.25 (2017-11-05)
@@ -18,7 +159,7 @@ History
   `#78 <https://github.com/wakatime/sublime-wakatime/issues/78>`_
 - Upgrade wakatime-cli to v9.0.0.
 - Detect project and branch names from git submodules.
-  `wakatime/#105 <https://github.com/wakatime/wakatime/issues/105>`_
+  `wakatime#105 <https://github.com/wakatime/wakatime/issues/105>`_
 
 
 7.0.24 (2017-10-29)
@@ -31,7 +172,7 @@ History
 - Upgrade requests to v2.18.4.
 - Upgrade tzlocal to v1.4.
 - Use WAKATIME_HOME env variable for offline and session caching.
-  `wakatime/#102 <https://github.com/wakatime/wakatime/issues/102>`_
+  `wakatime#102 <https://github.com/wakatime/wakatime/issues/102>`_
 
 
 7.0.23 (2017-09-14)

WakaTime.py

@@ -7,7 +7,7 @@ Website:     https://wakatime.com/
 ==========================================================="""
 
 
-__version__ = '7.0.26'
+__version__ = '8.3.2'
 
 
 import sublime
@@ -18,6 +18,7 @@ import json
 import os
 import platform
 import re
+import subprocess
 import sys
 import time
 import threading
@@ -25,7 +26,7 @@ import traceback
 import urllib
 import webbrowser
 from datetime import datetime
-from subprocess import Popen, STDOUT, PIPE
+from subprocess import STDOUT, PIPE
 from zipfile import ZipFile
 try:
     import _winreg as winreg  # py2
@@ -42,6 +43,8 @@ except ImportError:
 
 is_py2 = (sys.version_info[0] == 2)
 is_py3 = (sys.version_info[0] == 3)
+is_win = platform.system() == 'Windows'
+
 
 if is_py2:
     def u(text):
@@ -91,8 +94,22 @@ else:
     ))
 
 
+class Popen(subprocess.Popen):
+    """Patched Popen to prevent opening cmd window on Windows platform."""
+
+    def __init__(self, *args, **kwargs):
+        startupinfo = kwargs.get('startupinfo')
+        if is_win or True:
+            try:
+                startupinfo = startupinfo or subprocess.STARTUPINFO()
+                startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+            except AttributeError:
+                pass
+        kwargs['startupinfo'] = startupinfo
+        super(Popen, self).__init__(*args, **kwargs)
+
+
 # globals
-HEARTBEAT_FREQUENCY = 2
 ST_VERSION = int(sublime.version())
 PLUGIN_DIR = os.path.dirname(os.path.realpath(__file__))
 API_CLIENT = os.path.join(PLUGIN_DIR, 'packages', 'wakatime', 'cli.py')
@@ -103,8 +120,11 @@ LAST_HEARTBEAT = {
     'file': None,
     'is_write': False,
 }
+LAST_HEARTBEAT_SENT_AT = 0
 PYTHON_LOCATION = None
 HEARTBEATS = queue.Queue()
+HEARTBEAT_FREQUENCY = 2  # minutes between logging heartbeat when editing same file
+SEND_BUFFER_SECONDS = 30  # seconds between sending buffered heartbeats to API
 
 
 # Log Levels
@@ -305,13 +325,15 @@ def find_python_from_registry(location, reg=None):
     return val
 
 
-def find_python_in_folder(folder, headless=True):
+def find_python_in_folder(folder, python3=True, headless=True):
     pattern = re.compile(r'\d+\.\d+')
 
     path = 'python'
-    if folder is not None:
+    if folder:
         path = os.path.realpath(os.path.join(folder, 'python'))
-    if headless:
+    if python3:
+        path = u(path) + u('3')
+    elif headless:
         path = u(path) + u('w')
     log(DEBUG, u('Looking for Python at: {0}').format(u(path)))
     try:
@@ -325,9 +347,13 @@ def find_python_in_folder(folder, headless=True):
     except:
         log(DEBUG, u(sys.exc_info()[1]))
 
-    if headless:
-        path = find_python_in_folder(folder, headless=False)
-        if path is not None:
+    if python3:
+        path = find_python_in_folder(folder, python3=False, headless=headless)
+        if path:
+            return path
+    elif headless:
+        path = find_python_in_folder(folder, python3=python3, headless=False)
+        if path:
             return path
 
     return None
@@ -427,11 +453,18 @@ def append_heartbeat(entity, timestamp, is_write, view, project, folders):
     }
 
     # process the queue of heartbeats in the future
-    seconds = 4
-    set_timeout(process_queue, seconds)
+    set_timeout(lambda: process_queue(timestamp), SEND_BUFFER_SECONDS)
 
 
-def process_queue():
+def process_queue(timestamp):
+    global LAST_HEARTBEAT_SENT_AT
+
+    # Prevent sending heartbeats more often than SEND_BUFFER_SECONDS
+    now = int(time.time())
+    if timestamp != LAST_HEARTBEAT['time'] and LAST_HEARTBEAT_SENT_AT > now - SEND_BUFFER_SECONDS:
+        return
+    LAST_HEARTBEAT_SENT_AT = now
+
     try:
         heartbeat = HEARTBEATS.get_nowait()
     except queue.Empty:
@@ -536,19 +569,16 @@ class SendHeartbeatsThread(threading.Thread):
             if self.has_extra_heartbeats:
                 cmd.append('--extra-heartbeats')
                 stdin = PIPE
-                extra_heartbeats = [self.build_heartbeat(**x) for x in self.extra_heartbeats]
-                extra_heartbeats = json.dumps(extra_heartbeats)
+                extra_heartbeats = json.dumps([self.build_heartbeat(**x) for x in self.extra_heartbeats])
+                inp = "{0}\n".format(extra_heartbeats).encode('utf-8')
             else:
                 extra_heartbeats = None
                 stdin = None
+                inp = None
 
             log(DEBUG, ' '.join(obfuscate_apikey(cmd)))
             try:
                 process = Popen(cmd, stdin=stdin, stdout=PIPE, stderr=STDOUT)
-                inp = None
-                if self.has_extra_heartbeats:
-                    inp = "{0}\n".format(extra_heartbeats)
-                    inp = inp.encode('utf-8')
                 output, err = process.communicate(input=inp)
                 output = u(output)
                 retcode = process.poll()

packages/wakatime/__about__.py

@@ -1,7 +1,7 @@
 __title__ = 'wakatime'
 __description__ = 'Common interface to the WakaTime api.'
 __url__ = 'https://github.com/wakatime/wakatime'
-__version_info__ = ('9', '0', '1')
+__version_info__ = ('10', '4', '1')
 __version__ = '.'.join(__version_info__)
 __author__ = 'Alan Hamlett'
 __author_email__ = 'alan@wakatime.com'

packages/wakatime/api.py

@@ -0,0 +1,226 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.api
+    ~~~~~~~~~~~~
+
+    :copyright: (c) 2017 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+
+from __future__ import print_function
+
+import base64
+import logging
+import sys
+import traceback
+
+from .compat import u, is_py3, json
+from .constants import API_ERROR, AUTH_ERROR, SUCCESS, UNKNOWN_ERROR
+
+from .offlinequeue import Queue
+from .packages.requests.exceptions import RequestException
+from .session_cache import SessionCache
+from .utils import get_hostname, get_user_agent
+from .packages import tzlocal
+
+
+log = logging.getLogger('WakaTime')
+
+
+try:
+    from .packages import requests
+except ImportError:  # pragma: nocover
+    log.traceback(logging.ERROR)
+    print(traceback.format_exc())
+    log.error('Please upgrade Python to the latest version.')
+    print('Please upgrade Python to the latest version.')
+    sys.exit(UNKNOWN_ERROR)
+
+
+def send_heartbeats(heartbeats, args, configs, use_ntlm_proxy=False):
+    """Send heartbeats to WakaTime API.
+
+    Returns `SUCCESS` when heartbeat was sent, otherwise returns an error code.
+    """
+
+    if len(heartbeats) == 0:
+        return SUCCESS
+
+    api_url = args.api_url
+    if not api_url:
+        api_url = 'https://api.wakatime.com/api/v1/users/current/heartbeats.bulk'
+    log.debug('Sending heartbeats to api at %s' % api_url)
+    timeout = args.timeout
+    if not timeout:
+        timeout = 60
+
+    data = [h.sanitize().dict() for h in heartbeats]
+    log.debug(data)
+
+    # setup api request
+    request_body = json.dumps(data)
+    api_key = u(base64.b64encode(str.encode(args.key) if is_py3 else args.key))
+    auth = u('Basic {api_key}').format(api_key=api_key)
+    headers = {
+        'User-Agent': get_user_agent(args.plugin),
+        'Content-Type': 'application/json',
+        'Accept': 'application/json',
+        'Authorization': auth,
+    }
+
+    hostname = get_hostname(args)
+    if hostname:
+        headers['X-Machine-Name'] = u(hostname).encode('utf-8')
+
+    # add Olson timezone to request
+    try:
+        tz = tzlocal.get_localzone()
+    except:
+        tz = None
+    if tz:
+        headers['TimeZone'] = u(tz.zone).encode('utf-8')
+
+    session_cache = SessionCache()
+    session = session_cache.get()
+
+    should_try_ntlm = False
+    proxies = {}
+    if args.proxy:
+        if use_ntlm_proxy:
+            from .packages.requests_ntlm import HttpNtlmAuth
+            username = args.proxy.rsplit(':', 1)
+            password = ''
+            if len(username) == 2:
+                password = username[1]
+            username = username[0]
+            session.auth = HttpNtlmAuth(username, password, session)
+        else:
+            should_try_ntlm = '\\' in args.proxy
+            proxies['https'] = args.proxy
+
+    # send request to api
+    response, code = None, None
+    try:
+        response = session.post(api_url, data=request_body, headers=headers,
+                                proxies=proxies, timeout=timeout,
+                                verify=not args.nosslverify)
+    except RequestException:
+        if should_try_ntlm:
+            return send_heartbeats(heartbeats, args, configs, use_ntlm_proxy=True)
+        else:
+            exception_data = {
+                sys.exc_info()[0].__name__: u(sys.exc_info()[1]),
+            }
+            if log.isEnabledFor(logging.DEBUG):
+                exception_data['traceback'] = traceback.format_exc()
+            if args.offline:
+                queue = Queue(args, configs)
+                queue.push_many(heartbeats)
+                if log.isEnabledFor(logging.DEBUG):
+                    log.warn(exception_data)
+            else:
+                log.error(exception_data)
+
+    except:  # delete cached session when requests raises unknown exception
+        if should_try_ntlm:
+            return send_heartbeats(heartbeats, args, configs, use_ntlm_proxy=True)
+        else:
+            exception_data = {
+                sys.exc_info()[0].__name__: u(sys.exc_info()[1]),
+                'traceback': traceback.format_exc(),
+            }
+            if args.offline:
+                queue = Queue(args, configs)
+                queue.push_many(heartbeats)
+                log.warn(exception_data)
+
+    else:
+        code = response.status_code if response is not None else None
+        content = response.text if response is not None else None
+
+        if _success(code):
+            results = _get_results(response)
+            _process_server_results(heartbeats, code, content, results, args, configs)
+            session_cache.save(session)
+            return SUCCESS
+        else:
+            log.debug({
+                'response_code': code,
+                'response_text': content,
+            })
+
+        if should_try_ntlm:
+            return send_heartbeats(heartbeats, args, configs, use_ntlm_proxy=True)
+
+        _handle_unsent_heartbeats(heartbeats, code, content, args, configs)
+
+    session_cache.delete()
+    return AUTH_ERROR if code == 401 else API_ERROR
+
+
+def _process_server_results(heartbeats, code, content, results, args, configs):
+    log.debug({
+        'response_code': code,
+        'results': results,
+    })
+
+    for i in range(len(results)):
+        if len(heartbeats) <= i:
+            log.warn('Results from api not matching heartbeats sent.')
+            break
+
+        try:
+            c = results[i][1]
+        except:
+            log.traceback(logging.WARNING)
+            c = 0
+        try:
+            text = json.dumps(results[i][0])
+        except:
+            log.traceback(logging.WARNING)
+            text = ''
+        if not _success(c):
+            _handle_unsent_heartbeats([heartbeats[i]], c, text, args, configs)
+
+    leftover = len(heartbeats) - len(results)
+    if leftover > 0:
+        log.warn('Missing {0} results from api.'.format(leftover))
+        start = len(heartbeats) - leftover
+        _handle_unsent_heartbeats(heartbeats[start:], code, content, args, configs)
+
+
+def _handle_unsent_heartbeats(heartbeats, code, content, args, configs):
+    if args.offline:
+        if code == 400:
+            log.error({
+                'response_code': code,
+                'response_content': content,
+            })
+        else:
+            if log.isEnabledFor(logging.DEBUG):
+                log.warn({
+                    'response_code': code,
+                    'response_content': content,
+                })
+            queue = Queue(args, configs)
+            queue.push_many(heartbeats)
+    else:
+        log.error({
+            'response_code': code,
+            'response_content': content,
+        })
+
+
+def _get_results(response):
+    results = []
+    if response is not None:
+        try:
+            results = response.json()['responses']
+        except:
+            log.traceback(logging.WARNING)
+    return results
+
+
+def _success(code):
+    return code == requests.codes.created or code == requests.codes.accepted

packages/wakatime/arguments.py

@@ -20,7 +20,7 @@ import traceback
 from .__about__ import __version__
 from .compat import basestring
 from .configs import parseConfigFile
-from .constants import AUTH_ERROR
+from .constants import AUTH_ERROR, DEFAULT_SYNC_OFFLINE_ACTIVITY
 from .packages import argparse
 
 
@@ -45,93 +45,162 @@ class StoreWithoutQuotes(argparse.Action):
         setattr(namespace, self.dest, values)
 
 
-def parseArguments():
+def parse_arguments():
     """Parse command line arguments and configs from ~/.wakatime.cfg.
     Command line arguments take precedence over config file settings.
     Returns instances of ArgumentParser and SafeConfigParser.
     """
 
     # define supported command line arguments
-    parser = argparse.ArgumentParser(
-            description='Common interface for the WakaTime api.')
+    parser = argparse.ArgumentParser(description='Common interface for the ' +
+                                                 'WakaTime api.')
     parser.add_argument('--entity', dest='entity', metavar='FILE',
-            action=FileAction,
-            help='absolute path to file for the heartbeat; can also be a '+
-                 'url, domain, or app when --entity-type is not file')
+                        action=FileAction,
+                        help='Absolute path to file for the heartbeat. Can ' +
+                             'also be a url, domain or app when ' +
+                             '--entity-type is not file.')
     parser.add_argument('--file', dest='file', action=FileAction,
-            help=argparse.SUPPRESS)
+                        help=argparse.SUPPRESS)
     parser.add_argument('--key', dest='key', action=StoreWithoutQuotes,
-            help='your wakatime api key; uses api_key from '+
-                '~/.wakatime.cfg by default')
-    parser.add_argument('--write', dest='is_write',
-            action='store_true',
-            help='when set, tells api this heartbeat was triggered from '+
-                 'writing to a file')
+                        help='Your wakatime api key; uses api_key from ' +
+                             '~/.wakatime.cfg by default.')
+    parser.add_argument('--write', dest='is_write', action='store_true',
+                        help='When set, tells api this heartbeat was ' +
+                             'triggered from writing to a file.')
     parser.add_argument('--plugin', dest='plugin', action=StoreWithoutQuotes,
-            help='optional text editor plugin name and version '+
-                 'for User-Agent header')
+                        help='Optional text editor plugin name and version ' +
+                             'for User-Agent header.')
     parser.add_argument('--time', dest='timestamp', metavar='time',
-            type=float, action=StoreWithoutQuotes,
-            help='optional floating-point unix epoch timestamp; '+
-                 'uses current time by default')
+                        type=float, action=StoreWithoutQuotes,
+                        help='Optional floating-point unix epoch timestamp. ' +
+                             'Uses current time by default.')
     parser.add_argument('--lineno', dest='lineno', action=StoreWithoutQuotes,
-            help='optional line number; current line being edited')
-    parser.add_argument('--cursorpos', dest='cursorpos', action=StoreWithoutQuotes,
-            help='optional cursor position in the current file')
-    parser.add_argument('--entity-type', dest='entity_type', action=StoreWithoutQuotes,
-            help='entity type for this heartbeat. can be one of "file", '+
-                 '"domain", or "app"; defaults to file.')
+                        help='Optional line number. This is the current ' +
+                             'line being edited.')
+    parser.add_argument('--cursorpos', dest='cursorpos',
+                        action=StoreWithoutQuotes,
+                        help='Optional cursor position in the current file.')
+    parser.add_argument('--entity-type', dest='entity_type',
+                        action=StoreWithoutQuotes,
+                        help='Entity type for this heartbeat. Can be ' +
+                             '"file", "domain" or "app". Defaults to "file".')
+    parser.add_argument('--category', dest='category',
+                        action=StoreWithoutQuotes,
+                        help='Category of this heartbeat activity. Can be ' +
+                             '"coding", "building", "indexing", ' +
+                             '"debugging", "running tests", ' +
+                             '"manual testing", "browsing", ' +
+                             '"code reviewing" or "designing". ' +
+                             'Defaults to "coding".')
     parser.add_argument('--proxy', dest='proxy', action=StoreWithoutQuotes,
-                        help='optional proxy configuration. Supports HTTPS '+
-                        'and SOCKS proxies. For example: '+
-                        'https://user:pass@host:port or '+
-                        'socks5://user:pass@host:port or ' +
-                        'domain\\user:pass')
+                        help='Optional proxy configuration. Supports HTTPS '+
+                             'and SOCKS proxies. For example: '+
+                             'https://user:pass@host:port or '+
+                             'socks5://user:pass@host:port or ' +
+                             'domain\\user:pass')
     parser.add_argument('--no-ssl-verify', dest='nosslverify',
                         action='store_true',
-                        help='disables SSL certificate verification for HTTPS '+
-                        'requests. By default, SSL certificates are verified.')
+                        help='Disables SSL certificate verification for HTTPS '+
+                             'requests. By default, SSL certificates are ' +
+                             'verified.')
     parser.add_argument('--project', dest='project', action=StoreWithoutQuotes,
-            help='optional project name')
-    parser.add_argument('--alternate-project', dest='alternate_project', action=StoreWithoutQuotes,
-            help='optional alternate project name; auto-discovered project '+
-                 'takes priority')
-    parser.add_argument('--alternate-language', dest='alternate_language', action=StoreWithoutQuotes,
-            help=argparse.SUPPRESS)
-    parser.add_argument('--language', dest='language', action=StoreWithoutQuotes,
-            help='optional language name; if valid, takes priority over '+
-                 'auto-detected language')
-    parser.add_argument('--hostname', dest='hostname', action=StoreWithoutQuotes, help='hostname of '+
-                        'current machine.')
-    parser.add_argument('--disableoffline', dest='offline',
-            action='store_false',
-            help='disables offline time logging instead of queuing logged time')
+                        help='Optional project name.')
+    parser.add_argument('--alternate-project', dest='alternate_project',
+                        action=StoreWithoutQuotes,
+                        help='Optional alternate project name. ' +
+                             'Auto-discovered project takes priority.')
+    parser.add_argument('--alternate-language', dest='alternate_language',
+                        action=StoreWithoutQuotes,
+                        help=argparse.SUPPRESS)
+    parser.add_argument('--language', dest='language',
+                        action=StoreWithoutQuotes,
+                        help='Optional language name. If valid, takes ' +
+                             'priority over auto-detected language.')
+    parser.add_argument('--local-file', dest='local_file', metavar='FILE',
+                        action=FileAction,
+                        help='Absolute path to local file for the ' +
+                             'heartbeat. When --entity is a remote file, ' +
+                             'this local file will be used for stats and ' +
+                             'just the value of --entity sent with heartbeat.')
+    parser.add_argument('--hostname', dest='hostname',
+                        action=StoreWithoutQuotes,
+                        help='Hostname of current machine.')
+    parser.add_argument('--disable-offline', dest='offline',
+                        action='store_false',
+                        help='Disables offline time logging instead of ' +
+                             'queuing logged time.')
+    parser.add_argument('--disableoffline', dest='offline_deprecated',
+                        action='store_true',
+                        help=argparse.SUPPRESS)
+    parser.add_argument('--hide-file-names', dest='hide_file_names',
+                        action='store_true',
+                        help='Obfuscate filenames. Will not send file names ' +
+                             'to api.')
+    parser.add_argument('--hide-filenames', dest='hide_filenames',
+                        action='store_true',
+                        help=argparse.SUPPRESS)
     parser.add_argument('--hidefilenames', dest='hidefilenames',
-            action='store_true',
-            help='obfuscate file names; will not send file names to api')
+                        action='store_true',
+                        help=argparse.SUPPRESS)
+    parser.add_argument('--hide-project-names', dest='hide_project_names',
+                        action='store_true',
+                        help='Obfuscate project names. When a project ' +
+                             'folder is detected instead of using the ' +
+                             'folder name as the project, a ' +
+                             '.wakatime-project file is created with a ' +
+                             'random project name.')
     parser.add_argument('--exclude', dest='exclude', action='append',
-            help='filename patterns to exclude from logging; POSIX regex '+
-                 'syntax; can be used more than once')
+                        help='Filename patterns to exclude from logging. ' +
+                             'POSIX regex syntax. Can be used more than once.')
+    parser.add_argument('--exclude-unknown-project',
+                        dest='exclude_unknown_project', action='store_true',
+                        help='When set, any activity where the project ' +
+                             'cannot be detected will be ignored.')
     parser.add_argument('--include', dest='include', action='append',
-            help='filename patterns to log; when used in combination with '+
-                 '--exclude, files matching include will still be logged; '+
-                 'POSIX regex syntax; can be used more than once')
+                        help='Filename patterns to log. When used in ' +
+                             'combination with --exclude, files matching ' +
+                             'include will still be logged. POSIX regex ' +
+                             'syntax. Can be used more than once.')
+    parser.add_argument('--include-only-with-project-file',
+                        dest='include_only_with_project_file',
+                        action='store_true',
+                        help='Disables tracking folders unless they contain ' +
+                             'a .wakatime-project file. Defaults to false.')
     parser.add_argument('--ignore', dest='ignore', action='append',
-            help=argparse.SUPPRESS)
+                        help=argparse.SUPPRESS)
     parser.add_argument('--extra-heartbeats', dest='extra_heartbeats',
-            action='store_true',
-            help='reads extra heartbeats from STDIN as a JSON array until EOF')
+                        action='store_true',
+                        help='Reads extra heartbeats from STDIN as a JSON ' +
+                             'array until EOF.')
+    parser.add_argument('--log-file', dest='log_file',
+                        action=StoreWithoutQuotes,
+                        help='Defaults to ~/.wakatime.log.')
     parser.add_argument('--logfile', dest='logfile', action=StoreWithoutQuotes,
-            help='defaults to ~/.wakatime.log')
-    parser.add_argument('--apiurl', dest='api_url', action=StoreWithoutQuotes,
-            help='heartbeats api url; for debugging with a local server')
-    parser.add_argument('--timeout', dest='timeout', type=int, action=StoreWithoutQuotes,
-            help='number of seconds to wait when sending heartbeats to api; '+
-                 'defaults to 60 seconds')
+                        help=argparse.SUPPRESS)
+    parser.add_argument('--api-url', dest='api_url', action=StoreWithoutQuotes,
+                        help='Heartbeats api url. For debugging with a ' +
+                             'local server.')
+    parser.add_argument('--apiurl', dest='apiurl', action=StoreWithoutQuotes,
+                        help=argparse.SUPPRESS)
+    parser.add_argument('--timeout', dest='timeout', type=int,
+                        action=StoreWithoutQuotes,
+                        help='Number of seconds to wait when sending ' +
+                             'heartbeats to api. Defaults to 60 seconds.')
+    parser.add_argument('--sync-offline-activity',
+                        dest='sync_offline_activity',
+                        action=StoreWithoutQuotes,
+                        help='Amount of offline activity to sync from your ' +
+                             'local ~/.wakatime.db sqlite3 file to your ' +
+                             'WakaTime Dashboard before exiting. Can be ' +
+                             '"none" or a positive integer number. Defaults ' +
+                             'to 5, meaning for every heartbeat sent while ' +
+                             'online 5 offline heartbeats are synced. Can ' +
+                             'be used without --entity to only sync offline ' +
+                             'activity without generating new heartbeats.')
     parser.add_argument('--config', dest='config', action=StoreWithoutQuotes,
-            help='defaults to ~/.wakatime.cfg')
+                        help='Defaults to ~/.wakatime.cfg.')
     parser.add_argument('--verbose', dest='verbose', action='store_true',
-            help='turns on debug messages in log file')
+                        help='Turns on debug messages in log file.')
     parser.add_argument('--version', action='version', version=__version__)
 
     # parse command line arguments
@@ -158,23 +227,34 @@ def parseArguments():
             args.key = default_key
         else:
             try:
-                parser.error('Missing api key. Find your api key from wakatime.com/settings.')
+                parser.error('Missing api key. Find your api key from wakatime.com/settings/api-key.')
             except SystemExit:
                 raise SystemExit(AUTH_ERROR)
 
     is_valid = not not re.match(r'^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$', args.key, re.I)
     if not is_valid:
         try:
-            parser.error('Invalid api key. Find your api key from wakatime.com/settings.')
+            parser.error('Invalid api key. Find your api key from wakatime.com/settings/api-key.')
         except SystemExit:
             raise SystemExit(AUTH_ERROR)
 
     if not args.entity:
         if args.file:
             args.entity = args.file
-        else:
+        elif not args.sync_offline_activity or args.sync_offline_activity == 'none':
             parser.error('argument --entity is required')
 
+    if not args.sync_offline_activity:
+        args.sync_offline_activity = DEFAULT_SYNC_OFFLINE_ACTIVITY
+    if args.sync_offline_activity == 'none':
+        args.sync_offline_activity = 0
+    try:
+        args.sync_offline_activity = int(args.sync_offline_activity)
+        if args.sync_offline_activity < 0:
+            raise Exception('Error')
+    except:
+        parser.error('argument --sync-offline-activity must be "none" or an integer number')
+
     if not args.language and args.alternate_language:
         args.language = args.alternate_language
 
@@ -194,6 +274,8 @@ def parseArguments():
                     args.exclude.append(pattern)
         except TypeError:  # pragma: nocover
             pass
+    if not args.include_only_with_project_file and configs.has_option('settings', 'include_only_with_project_file'):
+        args.include_only_with_project_file = configs.get('settings', 'include_only_with_project_file')
     if not args.include:
         args.include = []
     if configs.has_option('settings', 'include'):
@@ -203,18 +285,12 @@ def parseArguments():
                     args.include.append(pattern)
         except TypeError:  # pragma: nocover
             pass
-    if args.hidefilenames:
-        args.hidefilenames = ['.*']
-    else:
-        args.hidefilenames = []
-        if configs.has_option('settings', 'hidefilenames'):
-            option = configs.get('settings', 'hidefilenames')
-            if option.strip().lower() == 'true':
-                args.hidefilenames = ['.*']
-            elif option.strip().lower() != 'false':
-                for pattern in option.split("\n"):
-                    if pattern.strip() != '':
-                        args.hidefilenames.append(pattern)
+    if not args.exclude_unknown_project and configs.has_option('settings', 'exclude_unknown_project'):
+        args.exclude_unknown_project = configs.getboolean('settings', 'exclude_unknown_project')
+    boolean_or_list('hide_file_names', args, configs, alternative_names=['hide_filenames', 'hidefilenames'])
+    boolean_or_list('hide_project_names', args, configs, alternative_names=['hide_projectnames', 'hideprojectnames'])
+    if args.offline_deprecated:
+        args.offline = False
     if args.offline and configs.has_option('settings', 'offline'):
         args.offline = configs.getboolean('settings', 'offline')
     if not args.proxy and configs.has_option('settings', 'proxy'):
@@ -235,11 +311,15 @@ def parseArguments():
         args.verbose = configs.getboolean('settings', 'verbose')
     if not args.verbose and configs.has_option('settings', 'debug'):
         args.verbose = configs.getboolean('settings', 'debug')
-    if not args.logfile and configs.has_option('settings', 'logfile'):
-        args.logfile = configs.get('settings', 'logfile')
-    if not args.logfile and os.environ.get('WAKATIME_HOME'):
+    if not args.log_file and args.logfile:
+        args.log_file = args.logfile
+    if not args.log_file and configs.has_option('settings', 'log_file'):
+        args.log_file = configs.get('settings', 'log_file')
+    if not args.log_file and os.environ.get('WAKATIME_HOME'):
         home = os.environ.get('WAKATIME_HOME')
-        args.logfile = os.path.join(os.path.expanduser(home), '.wakatime.log')
+        args.log_file = os.path.join(os.path.expanduser(home), '.wakatime.log')
+    if not args.api_url and args.apiurl:
+        args.api_url = args.apiurl
     if not args.api_url and configs.has_option('settings', 'api_url'):
         args.api_url = configs.get('settings', 'api_url')
     if not args.timeout and configs.has_option('settings', 'timeout'):
@@ -249,3 +329,30 @@ def parseArguments():
             print(traceback.format_exc())
 
     return args, configs
+
+
+def boolean_or_list(config_name, args, configs, alternative_names=[]):
+    """Get a boolean or list of regexes from args and configs."""
+
+    # when argument flag present, set to wildcard regex
+    for key in alternative_names:
+        if hasattr(args, key) and getattr(args, key):
+            setattr(args, config_name, ['.*'])
+            return
+
+    setattr(args, config_name, [])
+
+    option = None
+    alternative_names.insert(0, config_name)
+    for key in alternative_names:
+        if configs.has_option('settings', key):
+            option = configs.get('settings', key)
+            break
+
+    if option is not None:
+        if option.strip().lower() == 'true':
+            setattr(args, config_name, ['.*'])
+        elif option.strip().lower() != 'false':
+            for pattern in option.split("\n"):
+                if pattern.strip() != '':
+                    getattr(args, config_name).append(pattern)

packages/wakatime/compat.py

@@ -9,12 +9,17 @@
     :license: BSD, see LICENSE for more details.
 """
 
+
 import codecs
+import os
+import platform
+import subprocess
 import sys
 
 
 is_py2 = (sys.version_info[0] == 2)
 is_py3 = (sys.version_info[0] == 3)
+is_win = platform.system() == 'Windows'
 
 
 if is_py2:  # pragma: nocover
@@ -91,3 +96,27 @@ except ImportError:  # pragma: nocover
             name = _resolve_name(name[level:], package, level)
         __import__(name)
         return sys.modules[name]
+
+
+try:
+    from .packages import simplejson as json
+except (ImportError, SyntaxError):  # pragma: nocover
+    import json
+
+
+class Popen(subprocess.Popen):
+    """Patched Popen to prevent opening cmd window on Windows platform."""
+
+    def __init__(self, *args, **kwargs):
+        startupinfo = kwargs.get('startupinfo')
+        if is_win or True:
+            try:
+                startupinfo = startupinfo or subprocess.STARTUPINFO()
+                startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+            except AttributeError:
+                pass
+        kwargs['startupinfo'] = startupinfo
+        if 'env' not in kwargs:
+            kwargs['env'] = os.environ.copy()
+            kwargs['env']['LANG'] = 'en-US' if is_win else 'en_US.UTF-8'
+        subprocess.Popen.__init__(self, *args, **kwargs)

packages/wakatime/configs.py

@@ -21,7 +21,7 @@ from .constants import CONFIG_FILE_PARSE_ERROR
 
 try:
     import configparser
-except ImportError:
+except ImportError:  # pragma: nocover
     from .packages import configparser
 
 

packages/wakatime/constants.py

@@ -34,11 +34,6 @@ Exit code used when there was an unhandled exception.
 """
 UNKNOWN_ERROR = 105
 
-""" Malformed Heartbeat Error
-Exit code used when the JSON input from `--extra-heartbeats` is malformed.
-"""
-MALFORMED_HEARTBEAT_ERROR = 106
-
 """ Connection Error
 Exit code used when there was proxy or other problem connecting to the WakaTime
 API servers.
@@ -50,3 +45,12 @@ Files larger than this in bytes will not have a line count stat for performance.
 Default is 2MB.
 """
 MAX_FILE_SIZE_SUPPORTED = 2000000
+
+""" Default limit of number of offline heartbeats to sync before exiting."""
+DEFAULT_SYNC_OFFLINE_ACTIVITY = 100
+
+""" Number of heartbeats per api request.
+Even when sending more heartbeats, this is the number of heartbeats sent per
+individual https request to the WakaTime API.
+"""
+HEARTBEATS_PER_REQUEST = 50

packages/wakatime/dependencies/__init__.py

@@ -106,8 +106,8 @@ class DependencyParser(object):
         self.lexer = lexer
 
         if self.lexer:
-            module_name = self.lexer.__module__.rsplit('.', 1)[-1]
-            class_name = self.lexer.__class__.__name__.replace('Lexer', 'Parser', 1)
+            module_name = self.root_lexer.__module__.rsplit('.', 1)[-1]
+            class_name = self.root_lexer.__class__.__name__.replace('Lexer', 'Parser', 1)
         else:
             module_name = 'unknown'
             class_name = 'UnknownParser'
@@ -121,6 +121,12 @@ class DependencyParser(object):
         except ImportError:
             log.debug('Parsing dependencies not supported for {0}.{1}'.format(module_name, class_name))
 
+    @property
+    def root_lexer(self):
+        if hasattr(self.lexer, 'root_lexer'):
+            return self.lexer.root_lexer
+        return self.lexer
+
     def parse(self):
         if self.parser:
             plugin = self.parser(self.source_file, lexer=self.lexer)

packages/wakatime/dependencies/c_cpp.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
-    wakatime.languages.c_cpp
-    ~~~~~~~~~~~~~~~~~~~~~~~~
+    wakatime.dependencies.c_cpp
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Parse dependencies from C++ code.
 

packages/wakatime/dependencies/data.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
-    wakatime.languages.data
-    ~~~~~~~~~~~~~~~~~~~~~~~
+    wakatime.dependencies.data
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Parse dependencies from data files.
 

packages/wakatime/dependencies/dotnet.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
-    wakatime.languages.dotnet
-    ~~~~~~~~~~~~~~~~~~~~~~~~~
+    wakatime.dependencies.dotnet
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Parse dependencies from .NET code.
 

packages/wakatime/dependencies/elm.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.elm
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Elm code.
+
+    :copyright: (c) 2018 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+
+
+class ElmParser(TokenParser):
+    state = None
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        elif self.partial(token) == 'Class':
+            self._process_class(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_namespace(self, token, content):
+        self.state = content.strip()
+
+    def _process_class(self, token, content):
+        if self.state == 'import':
+            self.append(self._format(content))
+
+    def _process_text(self, token, content):
+        pass
+
+    def _process_other(self, token, content):
+        self.state = None
+
+    def _format(self, content):
+        return content.strip().split('.')[0].strip()

packages/wakatime/dependencies/go.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
-    wakatime.languages.go
-    ~~~~~~~~~~~~~~~~~~~~~
+    wakatime.dependencies.go
+    ~~~~~~~~~~~~~~~~~~~~~~~~
 
     Parse dependencies from Go code.
 

packages/wakatime/dependencies/haskell.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.haskell
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Haskell code.
+
+    :copyright: (c) 2018 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+
+
+class HaskellParser(TokenParser):
+    state = None
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Reserved':
+            self._process_reserved(token, content)
+        elif self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        elif self.partial(token) == 'Keyword':
+            self._process_keyword(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_reserved(self, token, content):
+        self.state = content.strip()
+
+    def _process_namespace(self, token, content):
+        if self.state == 'import':
+            self.append(self._format(content))
+
+    def _process_keyword(self, token, content):
+        if self.state != 'import' or content.strip() != 'qualified':
+            self.state = None
+
+    def _process_text(self, token, content):
+        pass
+
+    def _process_other(self, token, content):
+        self.state = None
+
+    def _format(self, content):
+        return content.strip().split('.')[0].strip()

packages/wakatime/dependencies/haxe.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.haxe
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Haxe code.
+
+    :copyright: (c) 2018 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+
+
+class HaxeParser(TokenParser):
+    exclude = [
+        r'^haxe$',
+    ]
+    state = None
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_namespace(self, token, content):
+        if self.state == 'import':
+            self.append(self._format(content))
+            self.state = None
+        else:
+            self.state = content
+
+    def _process_text(self, token, content):
+        pass
+
+    def _process_other(self, token, content):
+        self.state = None
+
+    def _format(self, content):
+        return content.strip()

packages/wakatime/dependencies/html.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 """
-    wakatime.languages.templates
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    wakatime.dependencies.html
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    Parse dependencies from Templates.
+    Parse dependencies from HTML.
 
     :copyright: (c) 2014 Alan Hamlett.
     :license: BSD, see LICENSE for more details.
@@ -69,7 +69,7 @@ KEYWORDS = [
 ]
 
 
-class HtmlDjangoParser(TokenParser):
+class HtmlParser(TokenParser):
     tags = []
     opening_tag = False
     getting_attrs = False
@@ -141,63 +141,3 @@ class HtmlDjangoParser(TokenParser):
             elif content.startswith('"') or content.startswith("'"):
                 if self.current_attr_value is None:
                     self.current_attr_value = content
-
-
-class VelocityHtmlParser(HtmlDjangoParser):
-    pass
-
-
-class MyghtyHtmlParser(HtmlDjangoParser):
-    pass
-
-
-class MasonParser(HtmlDjangoParser):
-    pass
-
-
-class MakoHtmlParser(HtmlDjangoParser):
-    pass
-
-
-class CheetahHtmlParser(HtmlDjangoParser):
-    pass
-
-
-class HtmlGenshiParser(HtmlDjangoParser):
-    pass
-
-
-class RhtmlParser(HtmlDjangoParser):
-    pass
-
-
-class HtmlPhpParser(HtmlDjangoParser):
-    pass
-
-
-class HtmlSmartyParser(HtmlDjangoParser):
-    pass
-
-
-class EvoqueHtmlParser(HtmlDjangoParser):
-    pass
-
-
-class ColdfusionHtmlParser(HtmlDjangoParser):
-    pass
-
-
-class LassoHtmlParser(HtmlDjangoParser):
-    pass
-
-
-class HandlebarsHtmlParser(HtmlDjangoParser):
-    pass
-
-
-class YamlJinjaParser(HtmlDjangoParser):
-    pass
-
-
-class TwigHtmlParser(HtmlDjangoParser):
-    pass

packages/wakatime/dependencies/javascript.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.javascript
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from JavaScript code.
+
+    :copyright: (c) 2018 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import re
+
+from . import TokenParser
+
+
+class JavascriptParser(TokenParser):
+    state = None
+    extension = re.compile(r'\.\w{1,4}$')
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Reserved':
+            self._process_reserved(token, content)
+        elif self.partial(token) == 'Single':
+            self._process_string(token, content)
+        elif self.partial(token) == 'Punctuation':
+            self._process_punctuation(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_reserved(self, token, content):
+        if self.state is None:
+            self.state = content
+
+    def _process_string(self, token, content):
+        if self.state == 'import':
+            self.append(self._format_module(content))
+        self.state = None
+
+    def _process_punctuation(self, token, content):
+        if content == ';':
+            self.state = None
+
+    def _process_other(self, token, content):
+        pass
+
+    def _format_module(self, content):
+        content = content.strip().strip('"').strip("'").strip()
+        content = content.split('/')[-1].split('\\')[-1]
+        content = self.extension.sub('', content, count=1)
+        return content
+
+
+class TypeScriptParser(JavascriptParser):
+    pass

packages/wakatime/dependencies/jvm.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
-    wakatime.languages.java
-    ~~~~~~~~~~~~~~~~~~~~~~~
+    wakatime.dependencies.java
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Parse dependencies from Java code.
 
@@ -43,7 +43,7 @@ class JavaParser(TokenParser):
             self._process_other(token, content)
 
     def _process_namespace(self, token, content):
-        if u(content) == u('import'):
+        if u(content).split() and u(content).split()[0] == u('import'):
             self.state = 'import'
 
         elif self.state == 'import':
@@ -94,3 +94,89 @@ class JavaParser(TokenParser):
 
     def _process_other(self, token, content):
         pass
+
+
+class KotlinParser(TokenParser):
+    state = None
+    exclude = [
+        r'^java\.',
+    ]
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Keyword':
+            self._process_keyword(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        elif self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_keyword(self, token, content):
+        self.state = content
+
+    def _process_text(self, token, content):
+        pass
+
+    def _process_namespace(self, token, content):
+        if self.state == 'import':
+            self.append(self._format(content))
+        self.state = None
+
+    def _process_other(self, token, content):
+        self.state = None
+
+    def _format(self, content):
+        content = content.split(u('.'))
+
+        if content[-1] == u('*'):
+            content = content[:len(content) - 1]
+
+        if len(content) == 0:
+            return None
+
+        if len(content) == 1:
+            return content[0]
+
+        return u('.').join(content[:2])
+
+
+class ScalaParser(TokenParser):
+    state = None
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Keyword':
+            self._process_keyword(token, content)
+        elif self.partial(token) == 'Text':
+            self._process_text(token, content)
+        elif self.partial(token) == 'Namespace':
+            self._process_namespace(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_keyword(self, token, content):
+        self.state = content
+
+    def _process_text(self, token, content):
+        pass
+
+    def _process_namespace(self, token, content):
+        if self.state == 'import':
+            self.append(self._format(content))
+        self.state = None
+
+    def _process_other(self, token, content):
+        self.state = None
+
+    def _format(self, content):
+        return content.strip().lstrip('__root__').strip('_').strip('.')

packages/wakatime/dependencies/objective.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.objective
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Objective-C and Swift code.
+
+    :copyright: (c) 2018 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import re
+
+from . import TokenParser
+
+
+class SwiftParser(TokenParser):
+    state = None
+    exclude = [
+        r'^foundation$',
+    ]
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Declaration':
+            self._process_declaration(token, content)
+        elif self.partial(token) == 'Class':
+            self._process_class(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_declaration(self, token, content):
+        if self.state is None:
+            self.state = content
+
+    def _process_class(self, token, content):
+        if self.state == 'import':
+            self.append(content)
+        self.state = None
+
+    def _process_other(self, token, content):
+        pass
+
+
+class ObjectiveCParser(TokenParser):
+    state = None
+    extension = re.compile(r'\.[mh]$')
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Preproc':
+            self._process_preproc(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_preproc(self, token, content):
+        if self.state:
+            self._process_import(token, content)
+
+        self.state = content
+
+    def _process_import(self, token, content):
+        if self.state == '#' and content.startswith('import '):
+            self.append(self._format(content))
+        self.state = None
+
+    def _process_other(self, token, content):
+        pass
+
+    def _format(self, content):
+        content = content.strip().lstrip('import ').strip()
+        content = content.strip('"').strip("'").strip()
+        content = content.strip('<').strip('>').strip()
+        content = content.split('/')[0]
+        content = self.extension.sub('', content, count=1)
+        return content

packages/wakatime/dependencies/php.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
-    wakatime.languages.php
-    ~~~~~~~~~~~~~~~~~~~~~~
+    wakatime.dependencies.php
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Parse dependencies from PHP code.
 
@@ -16,6 +16,10 @@ from ..compat import u
 class PhpParser(TokenParser):
     state = None
     parens = 0
+    exclude = [
+        r'^app$',
+        r'app\.php$',
+    ]
 
     def parse(self):
         for index, token, content in self.tokens:

packages/wakatime/dependencies/python.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
-    wakatime.languages.python
-    ~~~~~~~~~~~~~~~~~~~~~~~~~
+    wakatime.dependencies.python
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Parse dependencies from Python code.
 

packages/wakatime/dependencies/rust.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.dependencies.rust
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Rust code.
+
+    :copyright: (c) 2018 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+
+
+class RustParser(TokenParser):
+    state = None
+
+    def parse(self):
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if self.partial(token) == 'Keyword':
+            self._process_keyword(token, content)
+        elif self.partial(token) == 'Whitespace':
+            self._process_whitespace(token, content)
+        elif self.partial(token) == 'Name':
+            self._process_name(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_keyword(self, token, content):
+        if self.state == 'extern' and content == 'crate':
+            self.state = 'extern crate'
+        else:
+            self.state = content
+
+    def _process_whitespace(self, token, content):
+        pass
+
+    def _process_name(self, token, content):
+        if self.state == 'extern crate':
+            self.append(content)
+        self.state = None
+
+    def _process_other(self, token, content):
+        self.state = None

packages/wakatime/dependencies/unknown.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
-    wakatime.languages.unknown
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+    wakatime.dependencies.unknown
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Parse dependencies from files of unknown language.
 

packages/wakatime/exceptions.py

@@ -12,3 +12,8 @@
 
 class NotYetImplemented(Exception):
     """This method needs to be implemented."""
+
+
+class SkipHeartbeat(Exception):
+    """Raised to prevent the current heartbeat from being sent."""
+    pass

packages/wakatime/heartbeat.py

@@ -0,0 +1,346 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.heartbeat
+    ~~~~~~~~~~~~~~~~~~
+    :copyright: (c) 2017 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+
+import os
+import logging
+import re
+from subprocess import PIPE
+
+from .compat import u, json, is_win, Popen
+from .exceptions import SkipHeartbeat
+from .project import get_project_info
+from .stats import get_file_stats
+from .utils import get_user_agent, should_exclude, format_file_path, find_project_file
+
+
+log = logging.getLogger('WakaTime')
+
+
+class Heartbeat(object):
+    """Heartbeat data for sending to API or storing in offline cache."""
+
+    skip = False
+    args = None
+    configs = None
+
+    time = None
+    entity = None
+    type = None
+    category = None
+    is_write = None
+    project = None
+    branch = None
+    language = None
+    dependencies = None
+    lines = None
+    lineno = None
+    cursorpos = None
+    user_agent = None
+
+    _sensitive = ('dependencies', 'lines', 'lineno', 'cursorpos', 'branch')
+
+    def __init__(self, data, args, configs, _clone=None):
+        if not data:
+            self.skip = u('Skipping because heartbeat data is missing.')
+            return
+
+        self.args = args
+        self.configs = configs
+
+        self.entity = data.get('entity')
+        self.time = data.get('time', data.get('timestamp'))
+        self.is_write = data.get('is_write')
+        self.user_agent = data.get('user_agent') or get_user_agent(args.plugin)
+
+        self.type = data.get('type', data.get('entity_type'))
+        if self.type not in ['file', 'domain', 'app']:
+            self.type = 'file'
+
+        self.category = data.get('category')
+        allowed_categories = [
+            'coding',
+            'building',
+            'indexing',
+            'debugging',
+            'running tests',
+            'manual testing',
+            'browsing',
+            'code reviewing',
+            'designing',
+        ]
+        if self.category not in allowed_categories:
+            self.category = None
+
+        if not _clone:
+            exclude = self._excluded_by_pattern()
+            if exclude:
+                self.skip = u('Skipping because matches exclude pattern: {pattern}').format(
+                    pattern=u(exclude),
+                )
+                return
+            if self.type == 'file':
+                self.entity = format_file_path(self.entity)
+                self._format_local_file()
+                if not self._file_exists():
+                    self.skip = u('File does not exist; ignoring this heartbeat.')
+                    return
+                if self._excluded_by_missing_project_file():
+                    self.skip = u('Skipping because missing .wakatime-project file in parent path.')
+                    return
+
+            if args.local_file and not os.path.isfile(args.local_file):
+                args.local_file = None
+
+            project, branch = get_project_info(configs, self, data)
+            self.project = project
+            self.branch = branch
+
+            if self._excluded_by_unknown_project():
+                self.skip = u('Skipping because project unknown.')
+                return
+
+            try:
+                stats = get_file_stats(self.entity,
+                                       entity_type=self.type,
+                                       lineno=data.get('lineno'),
+                                       cursorpos=data.get('cursorpos'),
+                                       plugin=args.plugin,
+                                       language=data.get('language'),
+                                       local_file=args.local_file)
+            except SkipHeartbeat as ex:
+                self.skip = u(ex) or 'Skipping'
+                return
+
+        else:
+            self.project = data.get('project')
+            self.branch = data.get('branch')
+            stats = data
+
+        for key in ['language', 'dependencies', 'lines', 'lineno', 'cursorpos']:
+            if stats.get(key) is not None:
+                setattr(self, key, stats[key])
+
+    def update(self, attrs):
+        """Return a copy of the current Heartbeat with updated attributes."""
+
+        data = self.dict()
+        data.update(attrs)
+        heartbeat = Heartbeat(data, self.args, self.configs, _clone=True)
+        return heartbeat
+
+    def sanitize(self):
+        """Removes sensitive data including file names and dependencies.
+
+        Returns a Heartbeat.
+        """
+
+        if not self.args.hide_file_names:
+            return self
+
+        if self.entity is None:
+            return self
+
+        if self.type != 'file':
+            return self
+
+        if self.should_obfuscate_filename():
+            self._sanitize_metadata()
+            extension = u(os.path.splitext(self.entity)[1])
+            self.entity = u('HIDDEN{0}').format(extension)
+        elif self.should_obfuscate_project():
+            self._sanitize_metadata()
+
+        return self
+
+    def json(self):
+        return json.dumps(self.dict())
+
+    def dict(self):
+        return {
+            'time': self.time,
+            'entity': self._unicode(self.entity),
+            'type': self.type,
+            'category': self.category,
+            'is_write': self.is_write,
+            'project': self._unicode(self.project),
+            'branch': self._unicode(self.branch),
+            'language': self._unicode(self.language),
+            'dependencies': self._unicode_list(self.dependencies),
+            'lines': self.lines,
+            'lineno': self.lineno,
+            'cursorpos': self.cursorpos,
+            'user_agent': self._unicode(self.user_agent),
+        }
+
+    def items(self):
+        return self.dict().items()
+
+    def get_id(self):
+        return u('{time}-{type}-{category}-{project}-{branch}-{entity}-{is_write}').format(
+            time=self.time,
+            type=self.type,
+            category=self.category,
+            project=self._unicode(self.project),
+            branch=self._unicode(self.branch),
+            entity=self._unicode(self.entity),
+            is_write=self.is_write,
+        )
+
+    def should_obfuscate_filename(self):
+        """Returns True if hide_file_names is true or the entity file path
+        matches one in the list of obfuscated file paths."""
+
+        for pattern in self.args.hide_file_names:
+            try:
+                compiled = re.compile(pattern, re.IGNORECASE)
+                if compiled.search(self.entity):
+                    return True
+            except re.error as ex:
+                log.warning(u('Regex error ({msg}) for hide_file_names pattern: {pattern}').format(
+                    msg=u(ex),
+                    pattern=u(pattern),
+                ))
+        return False
+
+    def should_obfuscate_project(self):
+        """Returns True if hide_project_names is true or the entity file path
+        matches one in the list of obfuscated project paths."""
+
+        for pattern in self.args.hide_project_names:
+            try:
+                compiled = re.compile(pattern, re.IGNORECASE)
+                if compiled.search(self.entity):
+                    return True
+            except re.error as ex:
+                log.warning(u('Regex error ({msg}) for hide_project_names pattern: {pattern}').format(
+                    msg=u(ex),
+                    pattern=u(pattern),
+                ))
+        return False
+
+    def _unicode(self, value):
+        if value is None:
+            return None
+        return u(value)
+
+    def _unicode_list(self, values):
+        if values is None:
+            return None
+        return [self._unicode(value) for value in values]
+
+    def _file_exists(self):
+        return (self.entity and os.path.isfile(self.entity) or
+            self.args.local_file and os.path.isfile(self.args.local_file))
+
+    def _format_local_file(self):
+        """When args.local_file empty on Windows, tries to map args.entity to a
+        unc path.
+
+        Updates args.local_file in-place without returning anything.
+        """
+
+        if self.type != 'file':
+            return
+
+        if not is_win:
+            return
+
+        if self._file_exists():
+            return
+
+        self.args.local_file = self._to_unc_path(self.entity)
+
+    def _to_unc_path(self, filepath):
+        drive, rest = self._splitdrive(filepath)
+        if not drive:
+            return filepath
+
+        stdout = None
+        try:
+            stdout, stderr = Popen(['net', 'use'], stdout=PIPE, stderr=PIPE).communicate()
+        except OSError:
+            pass
+        else:
+            if stdout:
+                cols = None
+                for line in stdout.strip().splitlines()[1:]:
+                    line = u(line)
+                    if not line.strip():
+                        continue
+                    if not cols:
+                        cols = self._unc_columns(line)
+                        continue
+                    start, end = cols.get('local', (0, 0))
+                    if not start and not end:
+                        break
+                    local = line[start:end].strip().split(':')[0].upper()
+                    if not local.isalpha():
+                        continue
+                    if local == drive:
+                        start, end = cols.get('remote', (0, 0))
+                        if not start and not end:
+                            break
+                        remote = line[start:end].strip()
+                        return remote + rest
+
+        return filepath
+
+    def _unc_columns(self, line):
+        cols = {}
+        current_col = u('')
+        newcol = False
+        start, end = 0, 0
+        for char in line:
+            if char.isalpha():
+                if newcol:
+                    cols[current_col.strip().lower()] = (start, end)
+                    current_col = u('')
+                    start = end
+                    newcol = False
+                current_col += u(char)
+            else:
+                newcol = True
+            end += 1
+        if start != end and current_col:
+            cols[current_col.strip().lower()] = (start, -1)
+        return cols
+
+    def _splitdrive(self, filepath):
+        if filepath[1:2] != ':' or not filepath[0].isalpha():
+            return None, filepath
+        return filepath[0].upper(), filepath[2:]
+
+    def _excluded_by_pattern(self):
+        return should_exclude(self.entity, self.args.include, self.args.exclude)
+
+    def _excluded_by_unknown_project(self):
+        if self.project:
+            return False
+        return self.args.exclude_unknown_project
+
+    def _excluded_by_missing_project_file(self):
+        if not self.args.include_only_with_project_file:
+            return False
+        return find_project_file(self.entity) is None
+
+    def _sanitize_metadata(self):
+        for key in self._sensitive:
+            setattr(self, key, None)
+
+    def __repr__(self):
+        return self.json()
+
+    def __bool__(self):
+        return not self.skip
+
+    def __nonzero__(self):
+        return self.__bool__()
+
+    def __getitem__(self, key):
+        return self.dict()[key]

packages/wakatime/languages/default.json

@@ -1,81 +1,81 @@
 {
-  "actionscript": "ActionScript", 
-  "apacheconf": "ApacheConf", 
-  "applescript": "AppleScript", 
-  "asp": "ASP", 
-  "assembly": "Assembly", 
-  "awk": "Awk", 
-  "bash": "Bash", 
-  "basic": "Basic", 
-  "brightscript": "BrightScript", 
-  "c": "C", 
-  "c#": "C#", 
-  "c++": "C++", 
-  "clojure": "Clojure", 
-  "cocoa": "Cocoa", 
-  "coffeescript": "CoffeeScript", 
-  "coldfusion": "ColdFusion", 
-  "common lisp": "Common Lisp", 
-  "cshtml": "CSHTML", 
-  "css": "CSS", 
-  "dart": "Dart", 
-  "delphi": "Delphi", 
-  "elixir": "Elixir", 
-  "elm": "Elm", 
-  "emacs lisp": "Emacs Lisp", 
-  "erlang": "Erlang", 
-  "f#": "F#", 
-  "fortran": "Fortran", 
-  "go": "Go", 
-  "gous": "Gosu", 
-  "groovy": "Groovy", 
-  "haml": "Haml", 
-  "haskell": "Haskell", 
-  "haxe": "Haxe", 
-  "html": "HTML", 
-  "ini": "INI", 
-  "jade": "Jade", 
-  "java": "Java", 
-  "javascript": "JavaScript", 
-  "json": "JSON", 
-  "jsx": "JSX", 
-  "kotlin": "Kotlin", 
-  "less": "LESS", 
-  "lua": "Lua", 
-  "markdown": "Markdown", 
-  "matlab": "Matlab", 
-  "mustache": "Mustache", 
-  "objective-c": "Objective-C", 
-  "objective-c++": "Objective-C++", 
-  "objective-j": "Objective-J", 
-  "ocaml": "OCaml", 
-  "perl": "Perl", 
-  "php": "PHP", 
-  "powershell": "PowerShell", 
-  "prolog": "Prolog", 
-  "puppet": "Puppet", 
-  "python": "Python", 
-  "r": "R", 
-  "restructuredtext": "reStructuredText", 
-  "ruby": "Ruby", 
-  "rust": "Rust", 
-  "sass": "Sass", 
-  "scala": "Scala", 
-  "scheme": "Scheme", 
-  "scss": "SCSS", 
-  "shell": "Shell", 
-  "slim": "Slim", 
-  "smalltalk": "Smalltalk", 
-  "sql": "SQL", 
-  "swift": "Swift", 
-  "text": "Text", 
-  "turing": "Turing", 
-  "twig": "Twig", 
-  "typescript": "TypeScript", 
-  "typoscript": "TypoScript", 
-  "vb.net": "VB.net", 
-  "viml": "VimL", 
-  "xaml": "XAML", 
-  "xml": "XML", 
+  "actionscript": "ActionScript",
+  "apacheconf": "ApacheConf",
+  "applescript": "AppleScript",
+  "asp": "ASP",
+  "assembly": "Assembly",
+  "awk": "Awk",
+  "bash": "Bash",
+  "basic": "Basic",
+  "brightscript": "BrightScript",
+  "c": "C",
+  "c#": "C#",
+  "c++": "C++",
+  "clojure": "Clojure",
+  "cocoa": "Cocoa",
+  "coffeescript": "CoffeeScript",
+  "coldfusion": "ColdFusion",
+  "common lisp": "Common Lisp",
+  "cshtml": "CSHTML",
+  "css": "CSS",
+  "dart": "Dart",
+  "delphi": "Delphi",
+  "elixir": "Elixir",
+  "elm": "Elm",
+  "emacs lisp": "Emacs Lisp",
+  "erlang": "Erlang",
+  "f#": "F#",
+  "fortran": "Fortran",
+  "go": "Go",
+  "gosu": "Gosu",
+  "groovy": "Groovy",
+  "haml": "Haml",
+  "haskell": "Haskell",
+  "haxe": "Haxe",
+  "html": "HTML",
+  "ini": "INI",
+  "jade": "Jade",
+  "java": "Java",
+  "javascript": "JavaScript",
+  "json": "JSON",
+  "jsx": "JSX",
+  "kotlin": "Kotlin",
+  "less": "LESS",
+  "lua": "Lua",
+  "markdown": "Markdown",
+  "matlab": "Matlab",
+  "mustache": "Mustache",
+  "objective-c": "Objective-C",
+  "objective-c++": "Objective-C++",
+  "objective-j": "Objective-J",
+  "ocaml": "OCaml",
+  "perl": "Perl",
+  "php": "PHP",
+  "powershell": "PowerShell",
+  "prolog": "Prolog",
+  "puppet": "Puppet",
+  "python": "Python",
+  "r": "R",
+  "restructuredtext": "reStructuredText",
+  "ruby": "Ruby",
+  "rust": "Rust",
+  "sass": "Sass",
+  "scala": "Scala",
+  "scheme": "Scheme",
+  "scss": "SCSS",
+  "shell": "Shell",
+  "slim": "Slim",
+  "smalltalk": "Smalltalk",
+  "sql": "SQL",
+  "swift": "Swift",
+  "text": "Text",
+  "turing": "Turing",
+  "twig": "Twig",
+  "typescript": "TypeScript",
+  "typoscript": "TypoScript",
+  "vb.net": "VB.net",
+  "viml": "VimL",
+  "xaml": "XAML",
+  "xml": "XML",
   "yaml": "YAML"
 }

packages/wakatime/logger.py

@@ -75,7 +75,7 @@ def setup_logging(args, version):
     for handler in logger.handlers:
         logger.removeHandler(handler)
     set_log_level(logger, args)
-    logfile = args.logfile
+    logfile = args.log_file
     if not logfile:
         logfile = '~/.wakatime.log'
     handler = logging.FileHandler(os.path.expanduser(logfile))

packages/wakatime/main.py

@@ -11,387 +11,76 @@
 
 from __future__ import print_function
 
-import base64
 import logging
 import os
-import re
 import sys
+import time
 import traceback
-import socket
 
 pwd = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, os.path.dirname(pwd))
 sys.path.insert(0, os.path.join(pwd, 'packages'))
 
 from .__about__ import __version__
-from .arguments import parseArguments
-from .compat import u, is_py3
-from .constants import (
-    API_ERROR,
-    AUTH_ERROR,
-    SUCCESS,
-    UNKNOWN_ERROR,
-    MALFORMED_HEARTBEAT_ERROR,
-)
+from .api import send_heartbeats
+from .arguments import parse_arguments
+from .compat import u, json
+from .constants import SUCCESS, UNKNOWN_ERROR, HEARTBEATS_PER_REQUEST
 from .logger import setup_logging
 
 log = logging.getLogger('WakaTime')
 
-try:
-    from .packages import requests
-except ImportError:
-    log.traceback(logging.ERROR)
-    print(traceback.format_exc())
-    log.error('Please upgrade Python to the latest version.')
-    print('Please upgrade Python to the latest version.')
-    sys.exit(UNKNOWN_ERROR)
-
+from .heartbeat import Heartbeat
 from .offlinequeue import Queue
-from .packages.requests.exceptions import RequestException
-from .project import get_project_info
-from .session_cache import SessionCache
-from .stats import get_file_stats
-from .utils import get_user_agent, should_exclude, format_file_path
-try:
-    from .packages import simplejson as json  # pragma: nocover
-except (ImportError, SyntaxError):  # pragma: nocover
-    import json
-from .packages import tzlocal
-
-
-def send_heartbeat(project=None, branch=None, hostname=None, stats={}, key=None,
-                   entity=None, timestamp=None, is_write=None, plugin=None,
-                   offline=None, entity_type='file', hidefilenames=None,
-                   proxy=None, nosslverify=None, api_url=None, timeout=None,
-                   use_ntlm_proxy=False, **kwargs):
-    """Sends heartbeat as POST request to WakaTime api server.
-
-    Returns `SUCCESS` when heartbeat was sent, otherwise returns an
-    error code constant.
-    """
-
-    if not api_url:
-        api_url = 'https://api.wakatime.com/api/v1/heartbeats'
-    if not timeout:
-        timeout = 60
-    log.debug('Sending heartbeat to api at %s' % api_url)
-    data = {
-        'time': timestamp,
-        'entity': entity,
-        'type': entity_type,
-    }
-    if stats.get('lines'):
-        data['lines'] = stats['lines']
-    if stats.get('language'):
-        data['language'] = stats['language']
-    if stats.get('dependencies'):
-        data['dependencies'] = stats['dependencies']
-    if stats.get('lineno'):
-        data['lineno'] = stats['lineno']
-    if stats.get('cursorpos'):
-        data['cursorpos'] = stats['cursorpos']
-    if is_write:
-        data['is_write'] = is_write
-    if project:
-        data['project'] = project
-    if branch:
-        data['branch'] = branch
-
-    if hidefilenames and entity is not None and entity_type == 'file':
-        for pattern in hidefilenames:
-            try:
-                compiled = re.compile(pattern, re.IGNORECASE)
-                if compiled.search(entity):
-                    extension = u(os.path.splitext(data['entity'])[1])
-                    data['entity'] = u('HIDDEN{0}').format(extension)
-
-                    # also delete any sensitive info when hiding file names
-                    sensitive = ['dependencies', 'lines', 'lineno', 'cursorpos', 'branch']
-                    for sensitiveKey in sensitive:
-                        if sensitiveKey in data:
-                            del data[sensitiveKey]
-
-                    break
-            except re.error as ex:
-                log.warning(u('Regex error ({msg}) for include pattern: {pattern}').format(
-                    msg=u(ex),
-                    pattern=u(pattern),
-                ))
-
-    log.debug(data)
-
-    # setup api request
-    request_body = json.dumps(data)
-    api_key = u(base64.b64encode(str.encode(key) if is_py3 else key))
-    auth = u('Basic {api_key}').format(api_key=api_key)
-    headers = {
-        'User-Agent': get_user_agent(plugin),
-        'Content-Type': 'application/json',
-        'Accept': 'application/json',
-        'Authorization': auth,
-    }
-    if hostname:
-        headers['X-Machine-Name'] = u(hostname).encode('utf-8')
-
-    # add Olson timezone to request
-    try:
-        tz = tzlocal.get_localzone()
-    except:
-        tz = None
-    if tz:
-        headers['TimeZone'] = u(tz.zone).encode('utf-8')
-
-    session_cache = SessionCache()
-    session = session_cache.get()
-
-    should_try_ntlm = False
-    proxies = {}
-    if proxy:
-        if use_ntlm_proxy:
-            from .packages.requests_ntlm import HttpNtlmAuth
-            username = proxy.rsplit(':', 1)
-            password = ''
-            if len(username) == 2:
-                password = username[1]
-            username = username[0]
-            session.auth = HttpNtlmAuth(username, password, session)
-        else:
-            should_try_ntlm = '\\' in proxy
-            proxies['https'] = proxy
-
-    # send request to api
-    response = None
-    try:
-        response = session.post(api_url, data=request_body, headers=headers,
-                                proxies=proxies, timeout=timeout,
-                                verify=not nosslverify)
-    except RequestException:
-        if should_try_ntlm:
-            return send_heartbeat(
-                project=project,
-                entity=entity,
-                timestamp=timestamp,
-                branch=branch,
-                hostname=hostname,
-                stats=stats,
-                key=key,
-                is_write=is_write,
-                plugin=plugin,
-                offline=offline,
-                hidefilenames=hidefilenames,
-                entity_type=entity_type,
-                proxy=proxy,
-                api_url=api_url,
-                timeout=timeout,
-                use_ntlm_proxy=True,
-            )
-        else:
-            exception_data = {
-                sys.exc_info()[0].__name__: u(sys.exc_info()[1]),
-            }
-            if log.isEnabledFor(logging.DEBUG):
-                exception_data['traceback'] = traceback.format_exc()
-            if offline:
-                queue = Queue()
-                queue.push(data, json.dumps(stats), plugin)
-                if log.isEnabledFor(logging.DEBUG):
-                    log.warn(exception_data)
-            else:
-                log.error(exception_data)
-
-    except:  # delete cached session when requests raises unknown exception
-        if should_try_ntlm:
-            return send_heartbeat(
-                project=project,
-                entity=entity,
-                timestamp=timestamp,
-                branch=branch,
-                hostname=hostname,
-                stats=stats,
-                key=key,
-                is_write=is_write,
-                plugin=plugin,
-                offline=offline,
-                hidefilenames=hidefilenames,
-                entity_type=entity_type,
-                proxy=proxy,
-                api_url=api_url,
-                timeout=timeout,
-                use_ntlm_proxy=True,
-            )
-        else:
-            exception_data = {
-                sys.exc_info()[0].__name__: u(sys.exc_info()[1]),
-                'traceback': traceback.format_exc(),
-            }
-            if offline:
-                queue = Queue()
-                queue.push(data, json.dumps(stats), plugin)
-                log.warn(exception_data)
-
-    else:
-        code = response.status_code if response is not None else None
-        content = response.text if response is not None else None
-        if code == requests.codes.created or code == requests.codes.accepted:
-            log.debug({
-                'response_code': code,
-            })
-            session_cache.save(session)
-            return SUCCESS
-        if should_try_ntlm:
-            return send_heartbeat(
-                project=project,
-                entity=entity,
-                timestamp=timestamp,
-                branch=branch,
-                hostname=hostname,
-                stats=stats,
-                key=key,
-                is_write=is_write,
-                plugin=plugin,
-                offline=offline,
-                hidefilenames=hidefilenames,
-                entity_type=entity_type,
-                proxy=proxy,
-                api_url=api_url,
-                timeout=timeout,
-                use_ntlm_proxy=True,
-            )
-        else:
-            if offline:
-                if code != 400:
-                    queue = Queue()
-                    queue.push(data, json.dumps(stats), plugin)
-                    if code == 401:
-                        log.error({
-                            'response_code': code,
-                            'response_content': content,
-                        })
-                        session_cache.delete()
-                        return AUTH_ERROR
-                    elif log.isEnabledFor(logging.DEBUG):
-                        log.warn({
-                            'response_code': code,
-                            'response_content': content,
-                        })
-                else:
-                    log.error({
-                        'response_code': code,
-                        'response_content': content,
-                    })
-            else:
-                log.error({
-                    'response_code': code,
-                    'response_content': content,
-                })
-    session_cache.delete()
-    return API_ERROR
-
-
-def sync_offline_heartbeats(args, hostname):
-    """Sends all heartbeats which were cached in the offline Queue."""
-
-    queue = Queue()
-    while True:
-        heartbeat = queue.pop()
-        if heartbeat is None:
-            break
-        status = send_heartbeat(
-            project=heartbeat['project'],
-            entity=heartbeat['entity'],
-            timestamp=heartbeat['time'],
-            branch=heartbeat['branch'],
-            hostname=hostname,
-            stats=json.loads(heartbeat['stats']),
-            key=args.key,
-            is_write=heartbeat['is_write'],
-            plugin=heartbeat['plugin'],
-            offline=args.offline,
-            hidefilenames=args.hidefilenames,
-            entity_type=heartbeat['type'],
-            proxy=args.proxy,
-            api_url=args.api_url,
-            timeout=args.timeout,
-        )
-        if status != SUCCESS:
-            if status == AUTH_ERROR:
-                return AUTH_ERROR
-            break
-    return SUCCESS
-
-
-def process_heartbeat(args, configs, hostname, heartbeat):
-    exclude = should_exclude(heartbeat['entity'], args.include, args.exclude)
-    if exclude is not False:
-        log.debug(u('Skipping because matches exclude pattern: {pattern}').format(
-            pattern=u(exclude),
-        ))
-        return SUCCESS
-
-    if heartbeat.get('entity_type') not in ['file', 'domain', 'app']:
-        heartbeat['entity_type'] = 'file'
-
-    if heartbeat['entity_type'] == 'file':
-        heartbeat['entity'] = format_file_path(heartbeat['entity'])
-
-    if heartbeat['entity_type'] != 'file' or os.path.isfile(heartbeat['entity']):
-
-        stats = get_file_stats(heartbeat['entity'],
-                               entity_type=heartbeat['entity_type'],
-                               lineno=heartbeat.get('lineno'),
-                               cursorpos=heartbeat.get('cursorpos'),
-                               plugin=args.plugin,
-                               language=heartbeat.get('language'))
-
-        project = heartbeat.get('project') or heartbeat.get('alternate_project')
-        branch = None
-        if heartbeat['entity_type'] == 'file':
-            project, branch = get_project_info(configs, heartbeat)
-
-        heartbeat['project'] = project
-        heartbeat['branch'] = branch
-        heartbeat['stats'] = stats
-        heartbeat['hostname'] = hostname
-        heartbeat['timeout'] = args.timeout
-        heartbeat['key'] = args.key
-        heartbeat['plugin'] = args.plugin
-        heartbeat['offline'] = args.offline
-        heartbeat['hidefilenames'] = args.hidefilenames
-        heartbeat['proxy'] = args.proxy
-        heartbeat['nosslverify'] = args.nosslverify
-        heartbeat['api_url'] = args.api_url
-
-        return send_heartbeat(**heartbeat)
-
-    else:
-        log.debug('File does not exist; ignoring this heartbeat.')
-        return SUCCESS
 
 
 def execute(argv=None):
     if argv:
         sys.argv = ['wakatime'] + argv
 
-    args, configs = parseArguments()
+    args, configs = parse_arguments()
 
     setup_logging(args, __version__)
 
     try:
+        heartbeats = []
 
-        hostname = args.hostname or socket.gethostname()
-
-        heartbeat = vars(args)
-        retval = process_heartbeat(args, configs, hostname, heartbeat)
+        hb = Heartbeat(vars(args), args, configs)
+        if hb:
+            heartbeats.append(hb)
+        else:
+            log.debug(hb.skip)
 
         if args.extra_heartbeats:
             try:
-                for heartbeat in json.loads(sys.stdin.readline()):
-                    retval = process_heartbeat(args, configs, hostname, heartbeat)
-            except json.JSONDecodeError:
-                retval = MALFORMED_HEARTBEAT_ERROR
+                for extra_data in json.loads(sys.stdin.readline()):
+                    hb = Heartbeat(extra_data, args, configs)
+                    if hb:
+                        heartbeats.append(hb)
+                    else:
+                        log.debug(hb.skip)
+            except json.JSONDecodeError as ex:
+                log.warning(u('Malformed extra heartbeats json: {msg}').format(
+                    msg=u(ex),
+                ))
+
+        retval = SUCCESS
+        while heartbeats:
+            retval = send_heartbeats(heartbeats[:HEARTBEATS_PER_REQUEST], args, configs)
+            heartbeats = heartbeats[HEARTBEATS_PER_REQUEST:]
+            if retval != SUCCESS:
+                break
+
+        if heartbeats:
+            Queue(args, configs).push_many(heartbeats)
 
         if retval == SUCCESS:
-            retval = sync_offline_heartbeats(args, hostname)
+            queue = Queue(args, configs)
+            for offline_heartbeats in queue.pop_many(args.sync_offline_activity):
+                time.sleep(1)
+                retval = send_heartbeats(offline_heartbeats, args, configs)
+                if retval != SUCCESS:
+                    break
 
         return retval
 

packages/wakatime/offlinequeue.py

@@ -14,77 +14,69 @@ import logging
 import os
 from time import sleep
 
+from .compat import json
+from .constants import DEFAULT_SYNC_OFFLINE_ACTIVITY, HEARTBEATS_PER_REQUEST
+from .heartbeat import Heartbeat
+
+
 try:
     import sqlite3
     HAS_SQL = True
 except ImportError:  # pragma: nocover
     HAS_SQL = False
 
-from .compat import u
-
 
 log = logging.getLogger('WakaTime')
 
 
 class Queue(object):
     db_file = '.wakatime.db'
-    table_name = 'heartbeat_1'
+    table_name = 'heartbeat_2'
 
-    def get_db_file(self):
-        home = '~'
-        if os.environ.get('WAKATIME_HOME'):
-            home = os.environ.get('WAKATIME_HOME')
-        return os.path.join(os.path.expanduser(home), '.wakatime.db')
+    args = None
+    configs = None
+
+    def __init__(self, args, configs):
+        self.args = args
+        self.configs = configs
 
     def connect(self):
-        conn = sqlite3.connect(self.get_db_file(), isolation_level=None)
+        conn = sqlite3.connect(self._get_db_file(), isolation_level=None)
         c = conn.cursor()
         c.execute('''CREATE TABLE IF NOT EXISTS {0} (
-            entity text,
-            type text,
-            time real,
-            project text,
-            branch text,
-            is_write integer,
-            stats text,
-            misc text,
-            plugin text)
+            id text,
+            heartbeat text)
         '''.format(self.table_name))
         return (conn, c)
 
-    def push(self, data, stats, plugin, misc=None):
-        if not HAS_SQL:  # pragma: nocover
+    def push(self, heartbeat):
+        if not HAS_SQL:
             return
         try:
             conn, c = self.connect()
-            heartbeat = {
-                'entity': u(data.get('entity')),
-                'type': u(data.get('type')),
-                'time': data.get('time'),
-                'project': u(data.get('project')),
-                'branch': u(data.get('branch')),
-                'is_write': 1 if data.get('is_write') else 0,
-                'stats': u(stats),
-                'misc': u(misc),
-                'plugin': u(plugin),
+            data = {
+                'id': heartbeat.get_id(),
+                'heartbeat': heartbeat.json(),
             }
-            c.execute('INSERT INTO {0} VALUES (:entity,:type,:time,:project,:branch,:is_write,:stats,:misc,:plugin)'.format(self.table_name), heartbeat)
+            c.execute('INSERT INTO {0} VALUES (:id,:heartbeat)'.format(self.table_name), data)
             conn.commit()
             conn.close()
         except sqlite3.Error:
             log.traceback()
 
     def pop(self):
-        if not HAS_SQL:  # pragma: nocover
+        if not HAS_SQL:
             return None
         tries = 3
         wait = 0.1
-        heartbeat = None
         try:
             conn, c = self.connect()
         except sqlite3.Error:
             log.traceback(logging.DEBUG)
             return None
+
+        heartbeat = None
+
         loop = True
         while loop and tries > -1:
             try:
@@ -92,40 +84,47 @@ class Queue(object):
                 c.execute('SELECT * FROM {0} LIMIT 1'.format(self.table_name))
                 row = c.fetchone()
                 if row is not None:
-                    values = []
-                    clauses = []
-                    index = 0
-                    for row_name in ['entity', 'type', 'time', 'project', 'branch', 'is_write']:
-                        if row[index] is not None:
-                            clauses.append('{0}=?'.format(row_name))
-                            values.append(row[index])
-                        else:  # pragma: nocover
-                            clauses.append('{0} IS NULL'.format(row_name))
-                        index += 1
-                    if len(values) > 0:
-                        c.execute('DELETE FROM {0} WHERE {1}'.format(self.table_name, ' AND '.join(clauses)), values)
-                    else:  # pragma: nocover
-                        c.execute('DELETE FROM {0} WHERE {1}'.format(self.table_name, ' AND '.join(clauses)))
+                    id = row[0]
+                    heartbeat = Heartbeat(json.loads(row[1]), self.args, self.configs, _clone=True)
+                    c.execute('DELETE FROM {0} WHERE id=?'.format(self.table_name), [id])
                 conn.commit()
-                if row is not None:
-                    heartbeat = {
-                        'entity': row[0],
-                        'type': row[1],
-                        'time': row[2],
-                        'project': row[3],
-                        'branch': row[4],
-                        'is_write': True if row[5] is 1 else False,
-                        'stats': row[6],
-                        'misc': row[7],
-                        'plugin': row[8],
-                    }
                 loop = False
-            except sqlite3.Error:  # pragma: nocover
+            except sqlite3.Error:
                 log.traceback(logging.DEBUG)
                 sleep(wait)
                 tries -= 1
         try:
             conn.close()
-        except sqlite3.Error:  # pragma: nocover
+        except sqlite3.Error:
             log.traceback(logging.DEBUG)
         return heartbeat
+
+    def push_many(self, heartbeats):
+        for heartbeat in heartbeats:
+            self.push(heartbeat)
+
+    def pop_many(self, limit=None):
+        if limit is None:
+            limit = DEFAULT_SYNC_OFFLINE_ACTIVITY
+
+        heartbeats = []
+
+        count = 0
+        while count < limit:
+            heartbeat = self.pop()
+            if not heartbeat:
+                break
+            heartbeats.append(heartbeat)
+            count += 1
+            if count % HEARTBEATS_PER_REQUEST == 0:
+                yield heartbeats
+                heartbeats = []
+
+        if heartbeats:
+            yield heartbeats
+
+    def _get_db_file(self):
+        home = '~'
+        if os.environ.get('WAKATIME_HOME'):
+            home = os.environ.get('WAKATIME_HOME')
+        return os.path.join(os.path.expanduser(home), '.wakatime.db')

packages/wakatime/packages/pygments/lexers/javascript.py

@@ -37,7 +37,7 @@ class JavascriptLexer(RegexLexer):
 
     name = 'JavaScript'
     aliases = ['js', 'javascript']
-    filenames = ['*.js', '*.jsm']
+    filenames = ['*.js', '*.jsm', '*.mjs']
     mimetypes = ['application/javascript', 'application/x-javascript',
                  'text/x-javascript', 'text/javascript']
 
@@ -1035,7 +1035,6 @@ class CoffeeScriptLexer(RegexLexer):
     filenames = ['*.coffee']
     mimetypes = ['text/coffeescript']
 
-
     _operator_re = (
         r'\+\+|~|&&|\band\b|\bor\b|\bis\b|\bisnt\b|\bnot\b|\?|:|'
         r'\|\||\\(?=\n)|'
@@ -1464,6 +1463,7 @@ class EarlGreyLexer(RegexLexer):
         ],
     }
 
+
 class JuttleLexer(RegexLexer):
     """
     For `Juttle`_ source code.

packages/wakatime/project.py

@@ -9,8 +9,11 @@
     :license: BSD, see LICENSE for more details.
 """
 
+import os
 import logging
+import random
 
+from .compat import open
 from .projects.git import Git
 from .projects.mercurial import Mercurial
 from .projects.projectfile import ProjectFile
@@ -33,7 +36,7 @@ REV_CONTROL_PLUGINS = [
 ]
 
 
-def get_project_info(configs, heartbeat):
+def get_project_info(configs, heartbeat, data):
     """Find the current project and branch.
 
     First looks for a .wakatime-project file. Second, uses the --project arg.
@@ -43,21 +46,29 @@ def get_project_info(configs, heartbeat):
     Returns a project, branch tuple.
     """
 
-    project_name, branch_name = None, None
+    project_name, branch_name = heartbeat.project, heartbeat.branch
 
-    for plugin_cls in CONFIG_PLUGINS:
+    if heartbeat.type != 'file':
+        project_name = project_name or heartbeat.args.project or heartbeat.args.alternate_project
+        return project_name, branch_name
 
-        plugin_name = plugin_cls.__name__.lower()
-        plugin_configs = get_configs_for_plugin(plugin_name, configs)
+    if project_name is None or branch_name is None:
 
-        project = plugin_cls(heartbeat['entity'], configs=plugin_configs)
-        if project.process():
-            project_name = project_name or project.name()
-            branch_name = project.branch()
-            break
+        for plugin_cls in CONFIG_PLUGINS:
+
+            plugin_name = plugin_cls.__name__.lower()
+            plugin_configs = get_configs_for_plugin(plugin_name, configs)
+
+            project = plugin_cls(heartbeat.entity, configs=plugin_configs)
+            if project.process():
+                project_name = project_name or project.name()
+                branch_name = project.branch()
+                break
 
     if project_name is None:
-        project_name = heartbeat.get('project')
+        project_name = data.get('project') or heartbeat.args.project
+
+    hide_project = heartbeat.should_obfuscate_project()
 
     if project_name is None or branch_name is None:
 
@@ -66,14 +77,23 @@ def get_project_info(configs, heartbeat):
             plugin_name = plugin_cls.__name__.lower()
             plugin_configs = get_configs_for_plugin(plugin_name, configs)
 
-            project = plugin_cls(heartbeat['entity'], configs=plugin_configs)
+            project = plugin_cls(heartbeat.entity, configs=plugin_configs)
             if project.process():
                 project_name = project_name or project.name()
                 branch_name = branch_name or project.branch()
+                if hide_project:
+                    branch_name = None
+                    project_name = generate_project_name()
+                    project_file = os.path.join(project.folder(), '.wakatime-project')
+                    try:
+                        with open(project_file, 'w') as fh:
+                            fh.write(project_name)
+                    except IOError:
+                        project_name = None
                 break
 
-    if project_name is None:
-        project_name = heartbeat.get('alternate_project')
+    if project_name is None and not hide_project:
+        project_name = data.get('alternate_project') or heartbeat.args.alternate_project
 
     return project_name, branch_name
 
@@ -82,3 +102,42 @@ def get_configs_for_plugin(plugin_name, configs):
     if configs and configs.has_section(plugin_name):
         return dict(configs.items(plugin_name))
     return None
+
+
+def generate_project_name():
+    """Generates a random project name."""
+
+    adjectives = [
+        'aged', 'ancient', 'autumn', 'billowing', 'bitter', 'black', 'blue', 'bold',
+        'broad', 'broken', 'calm', 'cold', 'cool', 'crimson', 'curly', 'damp',
+        'dark', 'dawn', 'delicate', 'divine', 'dry', 'empty', 'falling', 'fancy',
+        'flat', 'floral', 'fragrant', 'frosty', 'gentle', 'green', 'hidden', 'holy',
+        'icy', 'jolly', 'late', 'lingering', 'little', 'lively', 'long', 'lucky',
+        'misty', 'morning', 'muddy', 'mute', 'nameless', 'noisy', 'odd', 'old',
+        'orange', 'patient', 'plain', 'polished', 'proud', 'purple', 'quiet', 'rapid',
+        'raspy', 'red', 'restless', 'rough', 'round', 'royal', 'shiny', 'shrill',
+        'shy', 'silent', 'small', 'snowy', 'soft', 'solitary', 'sparkling', 'spring',
+        'square', 'steep', 'still', 'summer', 'super', 'sweet', 'throbbing', 'tight',
+        'tiny', 'twilight', 'wandering', 'weathered', 'white', 'wild', 'winter', 'wispy',
+        'withered', 'yellow', 'young'
+    ]
+    nouns = [
+        'art', 'band', 'bar', 'base', 'bird', 'block', 'boat', 'bonus',
+        'bread', 'breeze', 'brook', 'bush', 'butterfly', 'cake', 'cell', 'cherry',
+        'cloud', 'credit', 'darkness', 'dawn', 'dew', 'disk', 'dream', 'dust',
+        'feather', 'field', 'fire', 'firefly', 'flower', 'fog', 'forest', 'frog',
+        'frost', 'glade', 'glitter', 'grass', 'hall', 'hat', 'haze', 'heart',
+        'hill', 'king', 'lab', 'lake', 'leaf', 'limit', 'math', 'meadow',
+        'mode', 'moon', 'morning', 'mountain', 'mouse', 'mud', 'night', 'paper',
+        'pine', 'poetry', 'pond', 'queen', 'rain', 'recipe', 'resonance', 'rice',
+        'river', 'salad', 'scene', 'sea', 'shadow', 'shape', 'silence', 'sky',
+        'smoke', 'snow', 'snowflake', 'sound', 'star', 'sun', 'sun', 'sunset',
+        'surf', 'term', 'thunder', 'tooth', 'tree', 'truth', 'union', 'unit',
+        'violet', 'voice', 'water', 'waterfall', 'wave', 'wildflower', 'wind', 'wood'
+    ]
+    numbers = [str(x) for x in range(10)]
+    return ' '.join([
+        random.choice(adjectives).capitalize(),
+        random.choice(nouns).capitalize(),
+        random.choice(numbers) + random.choice(numbers),
+    ])

packages/wakatime/projects/base.py

@@ -43,3 +43,8 @@ class BaseProject(object):
         """ Returns the current branch.
         """
         raise NotYetImplemented()
+
+    def folder(self):
+        """ Returns the project's top folder path.
+        """
+        raise NotYetImplemented()

packages/wakatime/projects/git.py

@@ -25,6 +25,7 @@ class Git(BaseProject):
     _submodule = None
     _project_name = None
     _head_file = None
+    _project_folder = None
 
     def process(self):
         return self._find_git_config_file(self.path)
@@ -35,19 +36,14 @@ class Git(BaseProject):
     def branch(self):
         head = self._head_file
         if head:
-            try:
-                with open(head, 'r', encoding='utf-8') as fh:
-                    return self._get_branch_from_head_file(fh.readline())
-            except UnicodeDecodeError:  # pragma: nocover
-                try:
-                    with open(head, 'r', encoding=sys.getfilesystemencoding()) as fh:
-                        return self._get_branch_from_head_file(fh.readline())
-                except:
-                    log.traceback(logging.WARNING)
-            except IOError:  # pragma: nocover
-                log.traceback(logging.WARNING)
+            line = self._first_line_of_file(head)
+            if line is not None:
+                return self._get_branch_from_head_file(line)
         return u('master')
 
+    def folder(self):
+        return self._project_folder
+
     def _find_git_config_file(self, path):
         path = os.path.realpath(path)
         if os.path.isfile(path):
@@ -55,13 +51,26 @@ class Git(BaseProject):
         if os.path.isfile(os.path.join(path, '.git', 'config')):
             self._project_name = os.path.basename(path)
             self._head_file = os.path.join(path, '.git', 'HEAD')
+            self._project_folder = path
             return True
-        if self._submodules_supported_for_path(path):
-            submodule_path = self._find_path_from_submodule(path)
-            if submodule_path:
-                self._project_name = os.path.basename(path)
-                self._head_file = os.path.join(submodule_path, 'HEAD')
+
+        link_path = self._path_from_gitdir_link_file(path)
+        if link_path:
+
+            # first check if this is a worktree
+            if self._is_worktree(link_path):
+                self._project_name = self._project_from_worktree(link_path)
+                self._head_file = os.path.join(link_path, 'HEAD')
+                self._project_folder = path
                 return True
+
+            # next check if this is a submodule
+            if self._submodules_supported_for_path(path):
+                self._project_name = os.path.basename(path)
+                self._head_file = os.path.join(link_path, 'HEAD')
+                self._project_folder = path
+                return True
+
         split_path = os.path.split(path)
         if split_path[1] == '':
             return False
@@ -77,8 +86,6 @@ class Git(BaseProject):
             return True
 
         disabled = self._configs.get('submodules_disabled')
-        if not disabled:
-            return True
 
         if disabled.strip().lower() == 'true':
             return False
@@ -99,30 +106,51 @@ class Git(BaseProject):
 
         return True
 
-    def _find_path_from_submodule(self, path):
+    def _is_worktree(self, link_path):
+        return os.path.basename(os.path.dirname(link_path)) == 'worktrees'
+
+    def _path_from_gitdir_link_file(self, path):
         link = os.path.join(path, '.git')
         if not os.path.isfile(link):
             return None
 
+        line = self._first_line_of_file(link)
+        if line is not None:
+            return self._path_from_gitdir_string(path, line)
+
+        return None
+
+    def _path_from_gitdir_string(self, path, line):
+        if line.startswith('gitdir: '):
+            subpath = line[len('gitdir: '):].strip()
+            if os.path.isfile(os.path.join(path, subpath, 'HEAD')):
+                return os.path.realpath(os.path.join(path, subpath))
+
+        return None
+
+    def _project_from_worktree(self, link_path):
+        commondir = os.path.join(link_path, 'commondir')
+        if os.path.isfile(commondir):
+            line = self._first_line_of_file(commondir)
+            if line:
+                gitdir = os.path.abspath(os.path.join(link_path, line))
+                if os.path.basename(gitdir) == '.git':
+                    return os.path.basename(os.path.dirname(gitdir))
+
+        return None
+
+    def _first_line_of_file(self, filepath):
         try:
-            with open(link, 'r', encoding='utf-8') as fh:
-                return self._get_path_from_submodule_link(path, fh.readline())
+            with open(filepath, 'r', encoding='utf-8') as fh:
+                return fh.readline().strip()
         except UnicodeDecodeError:
-            try:
-                with open(link, 'r', encoding=sys.getfilesystemencoding()) as fh:
-                    return self._get_path_from_submodule_link(path, fh.readline())
-            except:
-                log.traceback(logging.WARNING)
+            pass
         except IOError:
+            pass
+        try:
+            with open(filepath, 'r', encoding=sys.getfilesystemencoding()) as fh:
+                return fh.readline().strip()
+        except:
             log.traceback(logging.WARNING)
 
         return None
-
-    def _get_path_from_submodule_link(self, path, line):
-        if line.startswith('gitdir: '):
-            subpath = line[len('gitdir: '):].strip()
-            if os.path.isfile(os.path.join(path, subpath, 'config')) and \
-                    os.path.isfile(os.path.join(path, subpath, 'HEAD')):
-                return os.path.join(path, subpath)
-
-        return None

packages/wakatime/projects/mercurial.py

@@ -47,6 +47,11 @@ class Mercurial(BaseProject):
                 log.traceback(logging.WARNING)
         return u('default')
 
+    def folder(self):
+        if self.configDir:
+            return os.path.dirname(self.configDir)
+        return None
+
     def _find_hg_config_dir(self, path):
         path = os.path.realpath(path)
         if os.path.isfile(path):

packages/wakatime/projects/projectfile.py

@@ -12,11 +12,11 @@
 """
 
 import logging
-import os
 import sys
 
 from .base import BaseProject
 from ..compat import u, open
+from ..utils import find_project_file
 
 
 log = logging.getLogger('WakaTime')
@@ -25,7 +25,7 @@ log = logging.getLogger('WakaTime')
 class ProjectFile(BaseProject):
 
     def process(self):
-        self.config = self._find_config(self.path)
+        self.config = find_project_file(self.path)
         self._project_name = None
         self._project_branch = None
 
@@ -33,13 +33,13 @@ class ProjectFile(BaseProject):
 
             try:
                 with open(self.config, 'r', encoding='utf-8') as fh:
-                    self._project_name = u(fh.readline().strip())
-                    self._project_branch = u(fh.readline().strip())
+                    self._project_name = u(fh.readline().strip()) or None
+                    self._project_branch = u(fh.readline().strip()) or None
             except UnicodeDecodeError:  # pragma: nocover
                 try:
                     with open(self.config, 'r', encoding=sys.getfilesystemencoding()) as fh:
-                        self._project_name = u(fh.readline().strip())
-                        self._project_branch = u(fh.readline().strip())
+                        self._project_name = u(fh.readline().strip()) or None
+                        self._project_branch = u(fh.readline().strip()) or None
                 except:
                     log.traceback(logging.WARNING)
             except IOError:  # pragma: nocover
@@ -53,14 +53,3 @@ class ProjectFile(BaseProject):
 
     def branch(self):
         return self._project_branch
-
-    def _find_config(self, path):
-        path = os.path.realpath(path)
-        if os.path.isfile(path):
-            path = os.path.split(path)[0]
-        if os.path.isfile(os.path.join(path, '.wakatime-project')):
-            return os.path.join(path, '.wakatime-project')
-        split_path = os.path.split(path)
-        if split_path[1] == '':
-            return None
-        return self._find_config(split_path[0])

packages/wakatime/projects/subversion.py

@@ -12,10 +12,10 @@
 import logging
 import os
 import platform
-from subprocess import Popen, PIPE
+from subprocess import PIPE
 
 from .base import BaseProject
-from ..compat import u, open
+from ..compat import u, open, Popen
 try:
     from collections import OrderedDict
 except ImportError:  # pragma: nocover
@@ -41,6 +41,11 @@ class Subversion(BaseProject):
             return None  # pragma: nocover
         return u(self.info['URL'].split('/')[-1].split('\\')[-1])
 
+    def folder(self):
+        if 'Repository Root' not in self.info:
+            return None
+        return self.info['Repository Root']
+
     def _find_binary(self):
         if self.binary_location:
             return self.binary_location
@@ -65,24 +70,22 @@ class Subversion(BaseProject):
         if not self._is_mac() or self._has_xcode_tools():
             stdout = None
             try:
-                os.environ['LANG'] = 'en_US'
-                stdout, stderr = Popen([
-                    self._find_binary(), 'info', os.path.realpath(path)
-                ], stdout=PIPE, stderr=PIPE).communicate()
+                stdout, stderr = Popen(
+                    [self._find_binary(), 'info', os.path.realpath(path)],
+                    stdout=PIPE,
+                    stderr=PIPE,
+                ).communicate()
             except OSError:
                 pass
             else:
                 if stdout:
                     for line in stdout.splitlines():
-                        line = u(line)
-                        line = line.split(': ', 1)
+                        line = u(line).split(': ', 1)
                         if len(line) == 2:
                             info[line[0]] = line[1]
         return info
 
     def _find_project_base(self, path, found=False):
-        if platform.system() == 'Windows':
-            return False  # pragma: nocover
         path = os.path.realpath(path)
         if os.path.isfile(path):
             path = os.path.split(path)[0]

packages/wakatime/session_cache.py

@@ -33,14 +33,8 @@ class SessionCache(object):
     db_file = '.wakatime.db'
     table_name = 'session'
 
-    def get_db_file(self):
-        home = '~'
-        if os.environ.get('WAKATIME_HOME'):
-            home = os.environ.get('WAKATIME_HOME')
-        return os.path.join(os.path.expanduser(home), '.wakatime.db')
-
     def connect(self):
-        conn = sqlite3.connect(self.get_db_file(), isolation_level=None)
+        conn = sqlite3.connect(self._get_db_file(), isolation_level=None)
         c = conn.cursor()
         c.execute('''CREATE TABLE IF NOT EXISTS {0} (
             value BLOB)
@@ -110,3 +104,9 @@ class SessionCache(object):
             conn.close()
         except:
             log.traceback(logging.DEBUG)
+
+    def _get_db_file(self):
+        home = '~'
+        if os.environ.get('WAKATIME_HOME'):
+            home = os.environ.get('WAKATIME_HOME')
+        return os.path.join(os.path.expanduser(home), '.wakatime.db')

packages/wakatime/stats.py

@@ -17,6 +17,7 @@ import sys
 from .compat import u, open
 from .constants import MAX_FILE_SIZE_SUPPORTED
 from .dependencies import DependencyParser
+from .exceptions import SkipHeartbeat
 from .language_priorities import LANGUAGES
 
 from .packages.pygments.lexers import (
@@ -39,7 +40,7 @@ log = logging.getLogger('WakaTime')
 
 
 def get_file_stats(file_name, entity_type='file', lineno=None, cursorpos=None,
-                   plugin=None, language=None):
+                   plugin=None, language=None, local_file=None):
     if entity_type != 'file':
         stats = {
             'language': None,
@@ -51,22 +52,24 @@ def get_file_stats(file_name, entity_type='file', lineno=None, cursorpos=None,
     else:
         language, lexer = standardize_language(language, plugin)
         if not language:
-            language, lexer = guess_language(file_name)
+            language, lexer = guess_language(file_name, local_file)
 
-        parser = DependencyParser(file_name, lexer)
+        language = use_root_language(language, lexer)
+
+        parser = DependencyParser(local_file or file_name, lexer)
         dependencies = parser.parse()
 
         stats = {
             'language': language,
             'dependencies': dependencies,
-            'lines': number_lines_in_file(file_name),
+            'lines': number_lines_in_file(local_file or file_name),
             'lineno': lineno,
             'cursorpos': cursorpos,
         }
     return stats
 
 
-def guess_language(file_name):
+def guess_language(file_name, local_file):
     """Guess lexer and language for a file.
 
     Returns a tuple of (language_str, lexer_obj).
@@ -78,14 +81,14 @@ def guess_language(file_name):
     if language:
         lexer = get_lexer(language)
     else:
-        lexer = smart_guess_lexer(file_name)
+        lexer = smart_guess_lexer(file_name, local_file)
         if lexer:
             language = u(lexer.name)
 
     return language, lexer
 
 
-def smart_guess_lexer(file_name):
+def smart_guess_lexer(file_name, local_file):
     """Guess Pygments lexer for a file.
 
     Looks for a vim modeline in file contents, then compares the accuracy
@@ -96,7 +99,7 @@ def smart_guess_lexer(file_name):
 
     text = get_file_head(file_name)
 
-    lexer1, accuracy1 = guess_lexer_using_filename(file_name, text)
+    lexer1, accuracy1 = guess_lexer_using_filename(local_file or file_name, text)
     lexer2, accuracy2 = guess_lexer_using_modeline(text)
 
     if lexer1:
@@ -118,6 +121,8 @@ def guess_lexer_using_filename(file_name, text):
 
     try:
         lexer = custom_pygments_guess_lexer_for_filename(file_name, text)
+    except SkipHeartbeat as ex:
+        raise SkipHeartbeat(u(ex))
     except:
         log.traceback(logging.DEBUG)
 
@@ -167,17 +172,29 @@ def get_language_from_extension(file_name):
 
     filepart, extension = os.path.splitext(file_name)
 
-    if re.match(r'\.h.*', extension, re.IGNORECASE) or re.match(r'\.c.*', extension, re.IGNORECASE):
+    if re.match(r'\.h.*$', extension, re.IGNORECASE) or re.match(r'\.c.*$', extension, re.IGNORECASE):
 
         if os.path.exists(u('{0}{1}').format(u(filepart), u('.c'))) or os.path.exists(u('{0}{1}').format(u(filepart), u('.C'))):
             return 'C'
 
+        if os.path.exists(u('{0}{1}').format(u(filepart), u('.m'))) or os.path.exists(u('{0}{1}').format(u(filepart), u('.M'))):
+            return 'Objective-C'
+
+        if os.path.exists(u('{0}{1}').format(u(filepart), u('.mm'))) or os.path.exists(u('{0}{1}').format(u(filepart), u('.MM'))):
+            return 'Objective-C++'
+
         available_extensions = extensions_in_same_folder(file_name)
         if '.cpp' in available_extensions:
             return 'C++'
         if '.c' in available_extensions:
             return 'C'
 
+    if re.match(r'\.m$', extension, re.IGNORECASE) and (os.path.exists(u('{0}{1}').format(u(filepart), u('.h'))) or os.path.exists(u('{0}{1}').format(u(filepart), u('.H')))):
+        return 'Objective-C'
+
+    if re.match(r'\.mm$', extension, re.IGNORECASE) and (os.path.exists(u('{0}{1}').format(u(filepart), u('.h'))) or os.path.exists(u('{0}{1}').format(u(filepart), u('.H')))):
+        return 'Objective-C++'
+
     return None
 
 
@@ -236,6 +253,13 @@ def get_lexer(language):
     return None
 
 
+def use_root_language(language, lexer):
+    if lexer and hasattr(lexer, 'root_lexer'):
+        return u(lexer.root_lexer.name)
+
+    return language
+
+
 def get_language_from_json(language, key):
     """Finds the given language in a json file."""
 
@@ -299,6 +323,12 @@ def custom_pygments_guess_lexer_for_filename(_fn, _text, **options):
             return lexer(**options)
         result.append(customize_lexer_priority(_fn, rv, lexer))
 
+    matlab = list(filter(lambda x: x[2].name.lower() == 'matlab', result))
+    if len(matlab) > 0:
+        objc = list(filter(lambda x: x[2].name.lower() == 'objective-c', result))
+        if objc and objc[0][0] == matlab[0][0]:
+            raise SkipHeartbeat('Skipping because not enough language accuracy.')
+
     def type_sort(t):
         # sort by:
         # - analyse score
@@ -322,7 +352,17 @@ def customize_lexer_priority(file_name, accuracy, lexer):
     elif lexer_name == 'matlab':
         available_extensions = extensions_in_same_folder(file_name)
         if '.mat' in available_extensions:
-            priority = 0.06
+            accuracy += 0.01
+        if '.h' not in available_extensions:
+            accuracy += 0.01
+    elif lexer_name == 'objective-c':
+        available_extensions = extensions_in_same_folder(file_name)
+        if '.mat' in available_extensions:
+            accuracy -= 0.01
+        else:
+            accuracy += 0.01
+        if '.h' in available_extensions:
+            accuracy += 0.01
 
     return (accuracy, priority, lexer)
 

packages/wakatime/utils.py

@@ -14,6 +14,7 @@ import platform
 import logging
 import os
 import re
+import socket
 import sys
 
 from .__about__ import __version__
@@ -23,6 +24,10 @@ from .compat import u
 log = logging.getLogger('WakaTime')
 
 
+BACKSLASH_REPLACE_PATTERN = re.compile(r'[\\/]+')
+WINDOWS_DRIVE_PATTERN = re.compile(r'^[a-z]:/')
+
+
 def should_exclude(entity, include, exclude):
     if entity is not None and entity.strip() != '':
         for pattern in include:
@@ -48,7 +53,7 @@ def should_exclude(entity, include, exclude):
     return False
 
 
-def get_user_agent(plugin):
+def get_user_agent(plugin=None):
     ver = sys.version_info
     python_version = '%d.%d.%d.%s.%d' % (ver[0], ver[1], ver[2], ver[3], ver[4])
     user_agent = u('wakatime/{ver} ({platform}) Python{py_ver}').format(
@@ -73,7 +78,25 @@ def format_file_path(filepath):
 
     try:
         filepath = os.path.realpath(os.path.abspath(filepath))
-        filepath = re.sub(r'[/\\]', os.path.sep, filepath)
-    except:  # pragma: nocover
+        filepath = re.sub(BACKSLASH_REPLACE_PATTERN, '/', filepath)
+        if WINDOWS_DRIVE_PATTERN.match(filepath):
+            filepath = filepath.capitalize()
+    except:
         pass
     return filepath
+
+
+def get_hostname(args):
+    return args.hostname or socket.gethostname()
+
+
+def find_project_file(path):
+    path = os.path.realpath(path)
+    if os.path.isfile(path):
+        path = os.path.split(path)[0]
+    if os.path.isfile(os.path.join(path, '.wakatime-project')):
+        return os.path.join(path, '.wakatime-project')
+    split_path = os.path.split(path)
+    if split_path[1] == '':
+        return None
+    return find_project_file(split_path[0])