v8.3.3

HISTORY.rst

@@ -3,6 +3,70 @@ History
 -------
 
 
+8.3.3 (2018-12-19)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.6.1.
+- Correctly parse include_only_with_project_file when set to false.
+  `wakatime#161 <https://github.com/wakatime/wakatime/issues/161>`_
+- Support language argument for non-file entity types.
+- Send 25 heartbeats per API request.
+- New category "Writing Tests".
+  `wakatime#156 <https://github.com/wakatime/wakatime/issues/156>`_
+- Fix bug caused by git config section without any submodule option defined.
+  `wakatime#152 <https://github.com/wakatime/wakatime/issues/152>`_
+
+
+8.3.2 (2018-10-06)
+++++++++++++++++++
+
+- Send buffered heartbeats to API every 30 seconds.
+
+
+8.3.1 (2018-10-05)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.4.1.
+- Send 50 offline heartbeats to API per request with 1 second delay in between.
+
+
+8.3.0 (2018-10-03)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.4.0.
+- Support logging coding activity to remote network drive files on Windows
+  platform by detecting UNC path from drive letter.
+  `wakatime#72 <https://github.com/wakatime/wakatime/issues/72>`_
+
+
+8.2.0 (2018-09-30)
+++++++++++++++++++
+
+- Prevent opening cmd window on Windows when running wakatime-cli.
+  `#91 <https://github.com/wakatime/sublime-wakatime/issues/91>`_
+- Upgrade wakatime-cli to v10.3.0.
+- Re-enable detecting projects from Subversion folder on Windows platform.
+- Prevent opening cmd window on Windows when detecting project from Subversion.
+- Run tests on Windows using Appveyor.
+
+
+8.1.2 (2018-09-20)
+++++++++++++++++++
+
+- Upgrade wakatime-cli to v10.2.4.
+- Default --sync-offline-activity to 100 instead of 5, so offline coding is
+  synced to dashboard faster.
+- Batch heartbeats in groups of 10 per api request.
+- New config hide_project_name and argument --hide-project-names for
+  obfuscating project names when sending coding activity to api.
+- Fix mispelled Gosu language.
+  `wakatime#137 <https://github.com/wakatime/wakatime/issues/137>`_
+- Remove metadata when hiding project or file names.
+- New --local-file argument to be used when --entity is a remote file.
+- New argument --sync-offline-activity for configuring the maximum offline
+  heartbeats to sync to the WakaTime API.
+
+
 8.1.1 (2018-04-26)
 ++++++++++++++++++
 
@@ -33,10 +97,10 @@ History
 - Upgrade wakatime-cli to v10.1.2.
 - Detect dependencies from Swift, Objective-C, TypeScript and JavaScript files.
 - Categorize .mjs files as JavaScript.
-  `#wakatime121 <https://github.com/wakatime/wakatime/issues/121>`_
+  `wakatime#121 <https://github.com/wakatime/wakatime/issues/121>`_
 - Detect dependencies from Elm, Haskell, Haxe, Kotlin, Rust, and Scala files.
 - Improved Matlab vs Objective-C language detection.
-  `#wakatime129 <https://github.com/wakatime/wakatime/issues/129>`_
+  `wakatime#129 <https://github.com/wakatime/wakatime/issues/129>`_
 
 
 8.0.6 (2018-01-04)

WakaTime.py

@@ -7,7 +7,7 @@ Website:     https://wakatime.com/
 ==========================================================="""
 
 
-__version__ = '8.1.1'
+__version__ = '8.3.3'
 
 
 import sublime
@@ -18,6 +18,7 @@ import json
 import os
 import platform
 import re
+import subprocess
 import sys
 import time
 import threading
@@ -25,7 +26,7 @@ import traceback
 import urllib
 import webbrowser
 from datetime import datetime
-from subprocess import Popen, STDOUT, PIPE
+from subprocess import STDOUT, PIPE
 from zipfile import ZipFile
 try:
     import _winreg as winreg  # py2
@@ -42,6 +43,8 @@ except ImportError:
 
 is_py2 = (sys.version_info[0] == 2)
 is_py3 = (sys.version_info[0] == 3)
+is_win = platform.system() == 'Windows'
+
 
 if is_py2:
     def u(text):
@@ -91,8 +94,22 @@ else:
     ))
 
 
+class Popen(subprocess.Popen):
+    """Patched Popen to prevent opening cmd window on Windows platform."""
+
+    def __init__(self, *args, **kwargs):
+        startupinfo = kwargs.get('startupinfo')
+        if is_win or True:
+            try:
+                startupinfo = startupinfo or subprocess.STARTUPINFO()
+                startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+            except AttributeError:
+                pass
+        kwargs['startupinfo'] = startupinfo
+        super(Popen, self).__init__(*args, **kwargs)
+
+
 # globals
-HEARTBEAT_FREQUENCY = 2
 ST_VERSION = int(sublime.version())
 PLUGIN_DIR = os.path.dirname(os.path.realpath(__file__))
 API_CLIENT = os.path.join(PLUGIN_DIR, 'packages', 'wakatime', 'cli.py')
@@ -103,8 +120,11 @@ LAST_HEARTBEAT = {
     'file': None,
     'is_write': False,
 }
+LAST_HEARTBEAT_SENT_AT = 0
 PYTHON_LOCATION = None
 HEARTBEATS = queue.Queue()
+HEARTBEAT_FREQUENCY = 2  # minutes between logging heartbeat when editing same file
+SEND_BUFFER_SECONDS = 30  # seconds between sending buffered heartbeats to API
 
 
 # Log Levels
@@ -433,11 +453,18 @@ def append_heartbeat(entity, timestamp, is_write, view, project, folders):
     }
 
     # process the queue of heartbeats in the future
-    seconds = 4
-    set_timeout(process_queue, seconds)
+    set_timeout(lambda: process_queue(timestamp), SEND_BUFFER_SECONDS)
 
 
-def process_queue():
+def process_queue(timestamp):
+    global LAST_HEARTBEAT_SENT_AT
+
+    # Prevent sending heartbeats more often than SEND_BUFFER_SECONDS
+    now = int(time.time())
+    if timestamp != LAST_HEARTBEAT['time'] and LAST_HEARTBEAT_SENT_AT > now - SEND_BUFFER_SECONDS:
+        return
+    LAST_HEARTBEAT_SENT_AT = now
+
     try:
         heartbeat = HEARTBEATS.get_nowait()
     except queue.Empty:
@@ -542,19 +569,16 @@ class SendHeartbeatsThread(threading.Thread):
             if self.has_extra_heartbeats:
                 cmd.append('--extra-heartbeats')
                 stdin = PIPE
-                extra_heartbeats = [self.build_heartbeat(**x) for x in self.extra_heartbeats]
-                extra_heartbeats = json.dumps(extra_heartbeats)
+                extra_heartbeats = json.dumps([self.build_heartbeat(**x) for x in self.extra_heartbeats])
+                inp = "{0}\n".format(extra_heartbeats).encode('utf-8')
             else:
                 extra_heartbeats = None
                 stdin = None
+                inp = None
 
             log(DEBUG, ' '.join(obfuscate_apikey(cmd)))
             try:
                 process = Popen(cmd, stdin=stdin, stdout=PIPE, stderr=STDOUT)
-                inp = None
-                if self.has_extra_heartbeats:
-                    inp = "{0}\n".format(extra_heartbeats)
-                    inp = inp.encode('utf-8')
                 output, err = process.communicate(input=inp)
                 output = u(output)
                 retcode = process.poll()

packages/wakatime/__about__.py

@@ -1,7 +1,7 @@
 __title__ = 'wakatime'
 __description__ = 'Common interface to the WakaTime api.'
 __url__ = 'https://github.com/wakatime/wakatime'
-__version_info__ = ('10', '2', '1')
+__version_info__ = ('10', '6', '1')
 __version__ = '.'.join(__version_info__)
 __author__ = 'Alan Hamlett'
 __author_email__ = 'alan@wakatime.com'

packages/wakatime/arguments.py

@@ -20,7 +20,7 @@ import traceback
 from .__about__ import __version__
 from .compat import basestring
 from .configs import parseConfigFile
-from .constants import AUTH_ERROR
+from .constants import AUTH_ERROR, DEFAULT_SYNC_OFFLINE_ACTIVITY
 from .packages import argparse
 
 
@@ -89,8 +89,8 @@ def parse_arguments():
                         help='Category of this heartbeat activity. Can be ' +
                              '"coding", "building", "indexing", ' +
                              '"debugging", "running tests", ' +
-                             '"manual testing", "browsing", ' +
-                             '"code reviewing" or "designing". ' +
+                             '"writing tests", "manual testing", ' +
+                             '"code reviewing", "browsing", or "designing". ' +
                              'Defaults to "coding".')
     parser.add_argument('--proxy', dest='proxy', action=StoreWithoutQuotes,
                         help='Optional proxy configuration. Supports HTTPS '+
@@ -116,6 +116,12 @@ def parse_arguments():
                         action=StoreWithoutQuotes,
                         help='Optional language name. If valid, takes ' +
                              'priority over auto-detected language.')
+    parser.add_argument('--local-file', dest='local_file', metavar='FILE',
+                        action=FileAction,
+                        help='Absolute path to local file for the ' +
+                             'heartbeat. When --entity is a remote file, ' +
+                             'this local file will be used for stats and ' +
+                             'just the value of --entity sent with heartbeat.')
     parser.add_argument('--hostname', dest='hostname',
                         action=StoreWithoutQuotes,
                         help='Hostname of current machine.')
@@ -126,13 +132,23 @@ def parse_arguments():
     parser.add_argument('--disableoffline', dest='offline_deprecated',
                         action='store_true',
                         help=argparse.SUPPRESS)
-    parser.add_argument('--hide-filenames', dest='hide_filenames',
+    parser.add_argument('--hide-file-names', dest='hide_file_names',
                         action='store_true',
                         help='Obfuscate filenames. Will not send file names ' +
                              'to api.')
+    parser.add_argument('--hide-filenames', dest='hide_filenames',
+                        action='store_true',
+                        help=argparse.SUPPRESS)
     parser.add_argument('--hidefilenames', dest='hidefilenames',
                         action='store_true',
                         help=argparse.SUPPRESS)
+    parser.add_argument('--hide-project-names', dest='hide_project_names',
+                        action='store_true',
+                        help='Obfuscate project names. When a project ' +
+                             'folder is detected instead of using the ' +
+                             'folder name as the project, a ' +
+                             '.wakatime-project file is created with a ' +
+                             'random project name.')
     parser.add_argument('--exclude', dest='exclude', action='append',
                         help='Filename patterns to exclude from logging. ' +
                              'POSIX regex syntax. Can be used more than once.')
@@ -170,6 +186,17 @@ def parse_arguments():
                         action=StoreWithoutQuotes,
                         help='Number of seconds to wait when sending ' +
                              'heartbeats to api. Defaults to 60 seconds.')
+    parser.add_argument('--sync-offline-activity',
+                        dest='sync_offline_activity',
+                        action=StoreWithoutQuotes,
+                        help='Amount of offline activity to sync from your ' +
+                             'local ~/.wakatime.db sqlite3 file to your ' +
+                             'WakaTime Dashboard before exiting. Can be ' +
+                             '"none" or a positive integer number. Defaults ' +
+                             'to 5, meaning for every heartbeat sent while ' +
+                             'online 5 offline heartbeats are synced. Can ' +
+                             'be used without --entity to only sync offline ' +
+                             'activity without generating new heartbeats.')
     parser.add_argument('--config', dest='config', action=StoreWithoutQuotes,
                         help='Defaults to ~/.wakatime.cfg.')
     parser.add_argument('--verbose', dest='verbose', action='store_true',
@@ -214,9 +241,20 @@ def parse_arguments():
     if not args.entity:
         if args.file:
             args.entity = args.file
-        else:
+        elif not args.sync_offline_activity or args.sync_offline_activity == 'none':
             parser.error('argument --entity is required')
 
+    if not args.sync_offline_activity:
+        args.sync_offline_activity = DEFAULT_SYNC_OFFLINE_ACTIVITY
+    if args.sync_offline_activity == 'none':
+        args.sync_offline_activity = 0
+    try:
+        args.sync_offline_activity = int(args.sync_offline_activity)
+        if args.sync_offline_activity < 0:
+            raise Exception('Error')
+    except:
+        parser.error('argument --sync-offline-activity must be "none" or an integer number')
+
     if not args.language and args.alternate_language:
         args.language = args.alternate_language
 
@@ -237,7 +275,7 @@ def parse_arguments():
         except TypeError:  # pragma: nocover
             pass
     if not args.include_only_with_project_file and configs.has_option('settings', 'include_only_with_project_file'):
-        args.include_only_with_project_file = configs.get('settings', 'include_only_with_project_file')
+        args.include_only_with_project_file = configs.get('settings', 'include_only_with_project_file') == 'true'
     if not args.include:
         args.include = []
     if configs.has_option('settings', 'include'):
@@ -249,24 +287,8 @@ def parse_arguments():
             pass
     if not args.exclude_unknown_project and configs.has_option('settings', 'exclude_unknown_project'):
         args.exclude_unknown_project = configs.getboolean('settings', 'exclude_unknown_project')
-    if not args.hide_filenames and args.hidefilenames:
-        args.hide_filenames = args.hidefilenames
-    if args.hide_filenames:
-        args.hide_filenames = ['.*']
-    else:
-        args.hide_filenames = []
-        option = None
-        if configs.has_option('settings', 'hidefilenames'):
-            option = configs.get('settings', 'hidefilenames')
-        if configs.has_option('settings', 'hide_filenames'):
-            option = configs.get('settings', 'hide_filenames')
-        if option is not None:
-            if option.strip().lower() == 'true':
-                args.hide_filenames = ['.*']
-            elif option.strip().lower() != 'false':
-                for pattern in option.split("\n"):
-                    if pattern.strip() != '':
-                        args.hide_filenames.append(pattern)
+    boolean_or_list('hide_file_names', args, configs, alternative_names=['hide_filenames', 'hidefilenames'])
+    boolean_or_list('hide_project_names', args, configs, alternative_names=['hide_projectnames', 'hideprojectnames'])
     if args.offline_deprecated:
         args.offline = False
     if args.offline and configs.has_option('settings', 'offline'):
@@ -307,3 +329,30 @@ def parse_arguments():
             print(traceback.format_exc())
 
     return args, configs
+
+
+def boolean_or_list(config_name, args, configs, alternative_names=[]):
+    """Get a boolean or list of regexes from args and configs."""
+
+    # when argument flag present, set to wildcard regex
+    for key in alternative_names:
+        if hasattr(args, key) and getattr(args, key):
+            setattr(args, config_name, ['.*'])
+            return
+
+    setattr(args, config_name, [])
+
+    option = None
+    alternative_names.insert(0, config_name)
+    for key in alternative_names:
+        if configs.has_option('settings', key):
+            option = configs.get('settings', key)
+            break
+
+    if option is not None:
+        if option.strip().lower() == 'true':
+            setattr(args, config_name, ['.*'])
+        elif option.strip().lower() != 'false':
+            for pattern in option.split("\n"):
+                if pattern.strip() != '':
+                    getattr(args, config_name).append(pattern)

packages/wakatime/compat.py

@@ -11,11 +11,15 @@
 
 
 import codecs
+import os
+import platform
+import subprocess
 import sys
 
 
 is_py2 = (sys.version_info[0] == 2)
 is_py3 = (sys.version_info[0] == 3)
+is_win = platform.system() == 'Windows'
 
 
 if is_py2:  # pragma: nocover
@@ -98,3 +102,21 @@ try:
     from .packages import simplejson as json
 except (ImportError, SyntaxError):  # pragma: nocover
     import json
+
+
+class Popen(subprocess.Popen):
+    """Patched Popen to prevent opening cmd window on Windows platform."""
+
+    def __init__(self, *args, **kwargs):
+        startupinfo = kwargs.get('startupinfo')
+        if is_win or True:
+            try:
+                startupinfo = startupinfo or subprocess.STARTUPINFO()
+                startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+            except AttributeError:
+                pass
+        kwargs['startupinfo'] = startupinfo
+        if 'env' not in kwargs:
+            kwargs['env'] = os.environ.copy()
+            kwargs['env']['LANG'] = 'en-US' if is_win else 'en_US.UTF-8'
+        subprocess.Popen.__init__(self, *args, **kwargs)

packages/wakatime/constants.py

@@ -45,3 +45,12 @@ Files larger than this in bytes will not have a line count stat for performance.
 Default is 2MB.
 """
 MAX_FILE_SIZE_SUPPORTED = 2000000
+
+""" Default limit of number of offline heartbeats to sync before exiting."""
+DEFAULT_SYNC_OFFLINE_ACTIVITY = 100
+
+""" Number of heartbeats per api request.
+Even when sending more heartbeats, this is the number of heartbeats sent per
+individual https request to the WakaTime API.
+"""
+HEARTBEATS_PER_REQUEST = 25

packages/wakatime/dependencies/__init__.py

@@ -131,5 +131,5 @@ class DependencyParser(object):
         if self.parser:
             plugin = self.parser(self.source_file, lexer=self.lexer)
             dependencies = plugin.parse()
-            return list(set(dependencies))
+            return list(filter(bool, set(dependencies)))
         return []

packages/wakatime/heartbeat.py

@@ -10,8 +10,9 @@
 import os
 import logging
 import re
+from subprocess import PIPE
 
-from .compat import u, json
+from .compat import u, json, is_win, Popen
 from .exceptions import SkipHeartbeat
 from .project import get_project_info
 from .stats import get_file_stats
@@ -42,6 +43,8 @@ class Heartbeat(object):
     cursorpos = None
     user_agent = None
 
+    _sensitive = ('dependencies', 'lines', 'lineno', 'cursorpos', 'branch')
+
     def __init__(self, data, args, configs, _clone=None):
         if not data:
             self.skip = u('Skipping because heartbeat data is missing.')
@@ -67,6 +70,7 @@ class Heartbeat(object):
             'debugging',
             'running tests',
             'manual testing',
+            'writing tests',
             'browsing',
             'code reviewing',
             'designing',
@@ -83,13 +87,17 @@ class Heartbeat(object):
                 return
             if self.type == 'file':
                 self.entity = format_file_path(self.entity)
-                if not self.entity or not os.path.isfile(self.entity):
+                self._format_local_file()
+                if not self._file_exists():
                     self.skip = u('File does not exist; ignoring this heartbeat.')
                     return
                 if self._excluded_by_missing_project_file():
                     self.skip = u('Skipping because missing .wakatime-project file in parent path.')
                     return
 
+            if args.local_file and not os.path.isfile(args.local_file):
+                args.local_file = None
+
             project, branch = get_project_info(configs, self, data)
             self.project = project
             self.branch = branch
@@ -104,7 +112,8 @@ class Heartbeat(object):
                                        lineno=data.get('lineno'),
                                        cursorpos=data.get('cursorpos'),
                                        plugin=args.plugin,
-                                       language=data.get('language'))
+                                       language=data.get('language'),
+                                       local_file=args.local_file)
             except SkipHeartbeat as ex:
                 self.skip = u(ex) or 'Skipping'
                 return
@@ -132,7 +141,7 @@ class Heartbeat(object):
         Returns a Heartbeat.
         """
 
-        if not self.args.hide_filenames:
+        if not self.args.hide_file_names:
             return self
 
         if self.entity is None:
@@ -141,29 +150,12 @@ class Heartbeat(object):
         if self.type != 'file':
             return self
 
-        for pattern in self.args.hide_filenames:
-            try:
-                compiled = re.compile(pattern, re.IGNORECASE)
-                if compiled.search(self.entity):
-
-                    sanitized = {}
-                    sensitive = ['dependencies', 'lines', 'lineno', 'cursorpos', 'branch']
-                    for key, val in self.items():
-                        if key in sensitive:
-                            sanitized[key] = None
-                        else:
-                            sanitized[key] = val
-
-                    extension = u(os.path.splitext(self.entity)[1])
-                    sanitized['entity'] = u('HIDDEN{0}').format(extension)
-
-                    return self.update(sanitized)
-
-            except re.error as ex:
-                log.warning(u('Regex error ({msg}) for include pattern: {pattern}').format(
-                    msg=u(ex),
-                    pattern=u(pattern),
-                ))
+        if self.should_obfuscate_filename():
+            self._sanitize_metadata()
+            extension = u(os.path.splitext(self.entity)[1])
+            self.entity = u('HIDDEN{0}').format(extension)
+        elif self.should_obfuscate_project():
+            self._sanitize_metadata()
 
         return self
 
@@ -201,6 +193,38 @@ class Heartbeat(object):
             is_write=self.is_write,
         )
 
+    def should_obfuscate_filename(self):
+        """Returns True if hide_file_names is true or the entity file path
+        matches one in the list of obfuscated file paths."""
+
+        for pattern in self.args.hide_file_names:
+            try:
+                compiled = re.compile(pattern, re.IGNORECASE)
+                if compiled.search(self.entity):
+                    return True
+            except re.error as ex:
+                log.warning(u('Regex error ({msg}) for hide_file_names pattern: {pattern}').format(
+                    msg=u(ex),
+                    pattern=u(pattern),
+                ))
+        return False
+
+    def should_obfuscate_project(self):
+        """Returns True if hide_project_names is true or the entity file path
+        matches one in the list of obfuscated project paths."""
+
+        for pattern in self.args.hide_project_names:
+            try:
+                compiled = re.compile(pattern, re.IGNORECASE)
+                if compiled.search(self.entity):
+                    return True
+            except re.error as ex:
+                log.warning(u('Regex error ({msg}) for hide_project_names pattern: {pattern}').format(
+                    msg=u(ex),
+                    pattern=u(pattern),
+                ))
+        return False
+
     def _unicode(self, value):
         if value is None:
             return None
@@ -211,6 +235,88 @@ class Heartbeat(object):
             return None
         return [self._unicode(value) for value in values]
 
+    def _file_exists(self):
+        return (self.entity and os.path.isfile(self.entity) or
+            self.args.local_file and os.path.isfile(self.args.local_file))
+
+    def _format_local_file(self):
+        """When args.local_file empty on Windows, tries to map args.entity to a
+        unc path.
+
+        Updates args.local_file in-place without returning anything.
+        """
+
+        if self.type != 'file':
+            return
+
+        if not is_win:
+            return
+
+        if self._file_exists():
+            return
+
+        self.args.local_file = self._to_unc_path(self.entity)
+
+    def _to_unc_path(self, filepath):
+        drive, rest = self._splitdrive(filepath)
+        if not drive:
+            return filepath
+
+        stdout = None
+        try:
+            stdout, stderr = Popen(['net', 'use'], stdout=PIPE, stderr=PIPE).communicate()
+        except OSError:
+            pass
+        else:
+            if stdout:
+                cols = None
+                for line in stdout.strip().splitlines()[1:]:
+                    line = u(line)
+                    if not line.strip():
+                        continue
+                    if not cols:
+                        cols = self._unc_columns(line)
+                        continue
+                    start, end = cols.get('local', (0, 0))
+                    if not start and not end:
+                        break
+                    local = line[start:end].strip().split(':')[0].upper()
+                    if not local.isalpha():
+                        continue
+                    if local == drive:
+                        start, end = cols.get('remote', (0, 0))
+                        if not start and not end:
+                            break
+                        remote = line[start:end].strip()
+                        return remote + rest
+
+        return filepath
+
+    def _unc_columns(self, line):
+        cols = {}
+        current_col = u('')
+        newcol = False
+        start, end = 0, 0
+        for char in line:
+            if char.isalpha():
+                if newcol:
+                    cols[current_col.strip().lower()] = (start, end)
+                    current_col = u('')
+                    start = end
+                    newcol = False
+                current_col += u(char)
+            else:
+                newcol = True
+            end += 1
+        if start != end and current_col:
+            cols[current_col.strip().lower()] = (start, -1)
+        return cols
+
+    def _splitdrive(self, filepath):
+        if filepath[1:2] != ':' or not filepath[0].isalpha():
+            return None, filepath
+        return filepath[0].upper(), filepath[2:]
+
     def _excluded_by_pattern(self):
         return should_exclude(self.entity, self.args.include, self.args.exclude)
 
@@ -224,6 +330,10 @@ class Heartbeat(object):
             return False
         return find_project_file(self.entity) is None
 
+    def _sanitize_metadata(self):
+        for key in self._sensitive:
+            setattr(self, key, None)
+
     def __repr__(self):
         return self.json()
 

packages/wakatime/languages/default.json

@@ -1,81 +1,81 @@
 {
-  "actionscript": "ActionScript", 
-  "apacheconf": "ApacheConf", 
-  "applescript": "AppleScript", 
-  "asp": "ASP", 
-  "assembly": "Assembly", 
-  "awk": "Awk", 
-  "bash": "Bash", 
-  "basic": "Basic", 
-  "brightscript": "BrightScript", 
-  "c": "C", 
-  "c#": "C#", 
-  "c++": "C++", 
-  "clojure": "Clojure", 
-  "cocoa": "Cocoa", 
-  "coffeescript": "CoffeeScript", 
-  "coldfusion": "ColdFusion", 
-  "common lisp": "Common Lisp", 
-  "cshtml": "CSHTML", 
-  "css": "CSS", 
-  "dart": "Dart", 
-  "delphi": "Delphi", 
-  "elixir": "Elixir", 
-  "elm": "Elm", 
-  "emacs lisp": "Emacs Lisp", 
-  "erlang": "Erlang", 
-  "f#": "F#", 
-  "fortran": "Fortran", 
-  "go": "Go", 
-  "gous": "Gosu", 
-  "groovy": "Groovy", 
-  "haml": "Haml", 
-  "haskell": "Haskell", 
-  "haxe": "Haxe", 
-  "html": "HTML", 
-  "ini": "INI", 
-  "jade": "Jade", 
-  "java": "Java", 
-  "javascript": "JavaScript", 
-  "json": "JSON", 
-  "jsx": "JSX", 
-  "kotlin": "Kotlin", 
-  "less": "LESS", 
-  "lua": "Lua", 
-  "markdown": "Markdown", 
-  "matlab": "Matlab", 
-  "mustache": "Mustache", 
-  "objective-c": "Objective-C", 
-  "objective-c++": "Objective-C++", 
-  "objective-j": "Objective-J", 
-  "ocaml": "OCaml", 
-  "perl": "Perl", 
-  "php": "PHP", 
-  "powershell": "PowerShell", 
-  "prolog": "Prolog", 
-  "puppet": "Puppet", 
-  "python": "Python", 
-  "r": "R", 
-  "restructuredtext": "reStructuredText", 
-  "ruby": "Ruby", 
-  "rust": "Rust", 
-  "sass": "Sass", 
-  "scala": "Scala", 
-  "scheme": "Scheme", 
-  "scss": "SCSS", 
-  "shell": "Shell", 
-  "slim": "Slim", 
-  "smalltalk": "Smalltalk", 
-  "sql": "SQL", 
-  "swift": "Swift", 
-  "text": "Text", 
-  "turing": "Turing", 
-  "twig": "Twig", 
-  "typescript": "TypeScript", 
-  "typoscript": "TypoScript", 
-  "vb.net": "VB.net", 
-  "viml": "VimL", 
-  "xaml": "XAML", 
-  "xml": "XML", 
+  "actionscript": "ActionScript",
+  "apacheconf": "ApacheConf",
+  "applescript": "AppleScript",
+  "asp": "ASP",
+  "assembly": "Assembly",
+  "awk": "Awk",
+  "bash": "Bash",
+  "basic": "Basic",
+  "brightscript": "BrightScript",
+  "c": "C",
+  "c#": "C#",
+  "c++": "C++",
+  "clojure": "Clojure",
+  "cocoa": "Cocoa",
+  "coffeescript": "CoffeeScript",
+  "coldfusion": "ColdFusion",
+  "common lisp": "Common Lisp",
+  "cshtml": "CSHTML",
+  "css": "CSS",
+  "dart": "Dart",
+  "delphi": "Delphi",
+  "elixir": "Elixir",
+  "elm": "Elm",
+  "emacs lisp": "Emacs Lisp",
+  "erlang": "Erlang",
+  "f#": "F#",
+  "fortran": "Fortran",
+  "go": "Go",
+  "gosu": "Gosu",
+  "groovy": "Groovy",
+  "haml": "Haml",
+  "haskell": "Haskell",
+  "haxe": "Haxe",
+  "html": "HTML",
+  "ini": "INI",
+  "jade": "Jade",
+  "java": "Java",
+  "javascript": "JavaScript",
+  "json": "JSON",
+  "jsx": "JSX",
+  "kotlin": "Kotlin",
+  "less": "LESS",
+  "lua": "Lua",
+  "markdown": "Markdown",
+  "matlab": "Matlab",
+  "mustache": "Mustache",
+  "objective-c": "Objective-C",
+  "objective-c++": "Objective-C++",
+  "objective-j": "Objective-J",
+  "ocaml": "OCaml",
+  "perl": "Perl",
+  "php": "PHP",
+  "powershell": "PowerShell",
+  "prolog": "Prolog",
+  "puppet": "Puppet",
+  "python": "Python",
+  "r": "R",
+  "restructuredtext": "reStructuredText",
+  "ruby": "Ruby",
+  "rust": "Rust",
+  "sass": "Sass",
+  "scala": "Scala",
+  "scheme": "Scheme",
+  "scss": "SCSS",
+  "shell": "Shell",
+  "slim": "Slim",
+  "smalltalk": "Smalltalk",
+  "sql": "SQL",
+  "swift": "Swift",
+  "text": "Text",
+  "turing": "Turing",
+  "twig": "Twig",
+  "typescript": "TypeScript",
+  "typoscript": "TypoScript",
+  "vb.net": "VB.net",
+  "viml": "VimL",
+  "xaml": "XAML",
+  "xml": "XML",
   "yaml": "YAML"
 }

packages/wakatime/main.py

@@ -14,6 +14,7 @@ from __future__ import print_function
 import logging
 import os
 import sys
+import time
 import traceback
 
 pwd = os.path.dirname(os.path.abspath(__file__))
@@ -24,7 +25,7 @@ from .__about__ import __version__
 from .api import send_heartbeats
 from .arguments import parse_arguments
 from .compat import u, json
-from .constants import SUCCESS, UNKNOWN_ERROR
+from .constants import SUCCESS, UNKNOWN_ERROR, HEARTBEATS_PER_REQUEST
 from .logger import setup_logging
 
 log = logging.getLogger('WakaTime')
@@ -63,12 +64,23 @@ def execute(argv=None):
                     msg=u(ex),
                 ))
 
-        retval = send_heartbeats(heartbeats, args, configs)
+        retval = SUCCESS
+        while heartbeats:
+            retval = send_heartbeats(heartbeats[:HEARTBEATS_PER_REQUEST], args, configs)
+            heartbeats = heartbeats[HEARTBEATS_PER_REQUEST:]
+            if retval != SUCCESS:
+                break
+
+        if heartbeats:
+            Queue(args, configs).push_many(heartbeats)
+
         if retval == SUCCESS:
             queue = Queue(args, configs)
-            offline_heartbeats = queue.pop_many()
-            if len(offline_heartbeats) > 0:
+            for offline_heartbeats in queue.pop_many(args.sync_offline_activity):
+                time.sleep(1)
                 retval = send_heartbeats(offline_heartbeats, args, configs)
+                if retval != SUCCESS:
+                    break
 
         return retval
 

packages/wakatime/offlinequeue.py

@@ -15,6 +15,7 @@ import os
 from time import sleep
 
 from .compat import json
+from .constants import DEFAULT_SYNC_OFFLINE_ACTIVITY, HEARTBEATS_PER_REQUEST
 from .heartbeat import Heartbeat
 
 
@@ -104,19 +105,23 @@ class Queue(object):
 
     def pop_many(self, limit=None):
         if limit is None:
-            limit = 5
+            limit = DEFAULT_SYNC_OFFLINE_ACTIVITY
 
         heartbeats = []
 
         count = 0
-        while limit == 0 or count < limit:
+        while count < limit:
             heartbeat = self.pop()
             if not heartbeat:
                 break
             heartbeats.append(heartbeat)
             count += 1
+            if count % HEARTBEATS_PER_REQUEST == 0:
+                yield heartbeats
+                heartbeats = []
 
-        return heartbeats
+        if heartbeats:
+            yield heartbeats
 
     def _get_db_file(self):
         home = '~'

packages/wakatime/project.py

@@ -9,8 +9,11 @@
     :license: BSD, see LICENSE for more details.
 """
 
+import os
 import logging
+import random
 
+from .compat import open
 from .projects.git import Git
 from .projects.mercurial import Mercurial
 from .projects.projectfile import ProjectFile
@@ -65,6 +68,8 @@ def get_project_info(configs, heartbeat, data):
     if project_name is None:
         project_name = data.get('project') or heartbeat.args.project
 
+    hide_project = heartbeat.should_obfuscate_project()
+
     if project_name is None or branch_name is None:
 
         for plugin_cls in REV_CONTROL_PLUGINS:
@@ -76,9 +81,18 @@ def get_project_info(configs, heartbeat, data):
             if project.process():
                 project_name = project_name or project.name()
                 branch_name = branch_name or project.branch()
+                if hide_project:
+                    branch_name = None
+                    project_name = generate_project_name()
+                    project_file = os.path.join(project.folder(), '.wakatime-project')
+                    try:
+                        with open(project_file, 'w') as fh:
+                            fh.write(project_name)
+                    except IOError:
+                        project_name = None
                 break
 
-    if project_name is None:
+    if project_name is None and not hide_project:
         project_name = data.get('alternate_project') or heartbeat.args.alternate_project
 
     return project_name, branch_name
@@ -88,3 +102,42 @@ def get_configs_for_plugin(plugin_name, configs):
     if configs and configs.has_section(plugin_name):
         return dict(configs.items(plugin_name))
     return None
+
+
+def generate_project_name():
+    """Generates a random project name."""
+
+    adjectives = [
+        'aged', 'ancient', 'autumn', 'billowing', 'bitter', 'black', 'blue', 'bold',
+        'broad', 'broken', 'calm', 'cold', 'cool', 'crimson', 'curly', 'damp',
+        'dark', 'dawn', 'delicate', 'divine', 'dry', 'empty', 'falling', 'fancy',
+        'flat', 'floral', 'fragrant', 'frosty', 'gentle', 'green', 'hidden', 'holy',
+        'icy', 'jolly', 'late', 'lingering', 'little', 'lively', 'long', 'lucky',
+        'misty', 'morning', 'muddy', 'mute', 'nameless', 'noisy', 'odd', 'old',
+        'orange', 'patient', 'plain', 'polished', 'proud', 'purple', 'quiet', 'rapid',
+        'raspy', 'red', 'restless', 'rough', 'round', 'royal', 'shiny', 'shrill',
+        'shy', 'silent', 'small', 'snowy', 'soft', 'solitary', 'sparkling', 'spring',
+        'square', 'steep', 'still', 'summer', 'super', 'sweet', 'throbbing', 'tight',
+        'tiny', 'twilight', 'wandering', 'weathered', 'white', 'wild', 'winter', 'wispy',
+        'withered', 'yellow', 'young'
+    ]
+    nouns = [
+        'art', 'band', 'bar', 'base', 'bird', 'block', 'boat', 'bonus',
+        'bread', 'breeze', 'brook', 'bush', 'butterfly', 'cake', 'cell', 'cherry',
+        'cloud', 'credit', 'darkness', 'dawn', 'dew', 'disk', 'dream', 'dust',
+        'feather', 'field', 'fire', 'firefly', 'flower', 'fog', 'forest', 'frog',
+        'frost', 'glade', 'glitter', 'grass', 'hall', 'hat', 'haze', 'heart',
+        'hill', 'king', 'lab', 'lake', 'leaf', 'limit', 'math', 'meadow',
+        'mode', 'moon', 'morning', 'mountain', 'mouse', 'mud', 'night', 'paper',
+        'pine', 'poetry', 'pond', 'queen', 'rain', 'recipe', 'resonance', 'rice',
+        'river', 'salad', 'scene', 'sea', 'shadow', 'shape', 'silence', 'sky',
+        'smoke', 'snow', 'snowflake', 'sound', 'star', 'sun', 'sun', 'sunset',
+        'surf', 'term', 'thunder', 'tooth', 'tree', 'truth', 'union', 'unit',
+        'violet', 'voice', 'water', 'waterfall', 'wave', 'wildflower', 'wind', 'wood'
+    ]
+    numbers = [str(x) for x in range(10)]
+    return ' '.join([
+        random.choice(adjectives).capitalize(),
+        random.choice(nouns).capitalize(),
+        random.choice(numbers) + random.choice(numbers),
+    ])

packages/wakatime/projects/base.py

@@ -43,3 +43,8 @@ class BaseProject(object):
         """ Returns the current branch.
         """
         raise NotYetImplemented()
+
+    def folder(self):
+        """ Returns the project's top folder path.
+        """
+        raise NotYetImplemented()

packages/wakatime/projects/git.py

@@ -25,6 +25,7 @@ class Git(BaseProject):
     _submodule = None
     _project_name = None
     _head_file = None
+    _project_folder = None
 
     def process(self):
         return self._find_git_config_file(self.path)
@@ -40,6 +41,9 @@ class Git(BaseProject):
                 return self._get_branch_from_head_file(line)
         return u('master')
 
+    def folder(self):
+        return self._project_folder
+
     def _find_git_config_file(self, path):
         path = os.path.realpath(path)
         if os.path.isfile(path):
@@ -47,6 +51,7 @@ class Git(BaseProject):
         if os.path.isfile(os.path.join(path, '.git', 'config')):
             self._project_name = os.path.basename(path)
             self._head_file = os.path.join(path, '.git', 'HEAD')
+            self._project_folder = path
             return True
 
         link_path = self._path_from_gitdir_link_file(path)
@@ -56,12 +61,14 @@ class Git(BaseProject):
             if self._is_worktree(link_path):
                 self._project_name = self._project_from_worktree(link_path)
                 self._head_file = os.path.join(link_path, 'HEAD')
+                self._project_folder = path
                 return True
 
             # next check if this is a submodule
             if self._submodules_supported_for_path(path):
                 self._project_name = os.path.basename(path)
                 self._head_file = os.path.join(link_path, 'HEAD')
+                self._project_folder = path
                 return True
 
         split_path = os.path.split(path)
@@ -80,10 +87,10 @@ class Git(BaseProject):
 
         disabled = self._configs.get('submodules_disabled')
 
+        if not disabled or disabled.strip().lower() == 'false':
+            return True
         if disabled.strip().lower() == 'true':
             return False
-        if disabled.strip().lower() == 'false':
-            return True
 
         for pattern in disabled.split("\n"):
             if pattern.strip():

packages/wakatime/projects/mercurial.py

@@ -47,6 +47,11 @@ class Mercurial(BaseProject):
                 log.traceback(logging.WARNING)
         return u('default')
 
+    def folder(self):
+        if self.configDir:
+            return os.path.dirname(self.configDir)
+        return None
+
     def _find_hg_config_dir(self, path):
         path = os.path.realpath(path)
         if os.path.isfile(path):

packages/wakatime/projects/subversion.py

@@ -12,10 +12,10 @@
 import logging
 import os
 import platform
-from subprocess import Popen, PIPE
+from subprocess import PIPE
 
 from .base import BaseProject
-from ..compat import u, open
+from ..compat import u, open, Popen
 try:
     from collections import OrderedDict
 except ImportError:  # pragma: nocover
@@ -41,6 +41,11 @@ class Subversion(BaseProject):
             return None  # pragma: nocover
         return u(self.info['URL'].split('/')[-1].split('\\')[-1])
 
+    def folder(self):
+        if 'Repository Root' not in self.info:
+            return None
+        return self.info['Repository Root']
+
     def _find_binary(self):
         if self.binary_location:
             return self.binary_location
@@ -65,24 +70,22 @@ class Subversion(BaseProject):
         if not self._is_mac() or self._has_xcode_tools():
             stdout = None
             try:
-                os.environ['LANG'] = 'en_US'
-                stdout, stderr = Popen([
-                    self._find_binary(), 'info', os.path.realpath(path)
-                ], stdout=PIPE, stderr=PIPE).communicate()
+                stdout, stderr = Popen(
+                    [self._find_binary(), 'info', os.path.realpath(path)],
+                    stdout=PIPE,
+                    stderr=PIPE,
+                ).communicate()
             except OSError:
                 pass
             else:
                 if stdout:
                     for line in stdout.splitlines():
-                        line = u(line)
-                        line = line.split(': ', 1)
+                        line = u(line).split(': ', 1)
                         if len(line) == 2:
                             info[line[0]] = line[1]
         return info
 
     def _find_project_base(self, path, found=False):
-        if platform.system() == 'Windows':
-            return False  # pragma: nocover
         path = os.path.realpath(path)
         if os.path.isfile(path):
             path = os.path.split(path)[0]

packages/wakatime/stats.py

@@ -40,36 +40,33 @@ log = logging.getLogger('WakaTime')
 
 
 def get_file_stats(file_name, entity_type='file', lineno=None, cursorpos=None,
-                   plugin=None, language=None):
-    if entity_type != 'file':
-        stats = {
-            'language': None,
-            'dependencies': [],
-            'lines': None,
-            'lineno': lineno,
-            'cursorpos': cursorpos,
-        }
-    else:
-        language, lexer = standardize_language(language, plugin)
+                   plugin=None, language=None, local_file=None):
+    """Returns a hash of information about the entity."""
+
+    language = standardize_language(language, plugin)
+    stats = {
+        'language': language,
+        'dependencies': [],
+        'lines': None,
+        'lineno': lineno,
+        'cursorpos': cursorpos,
+    }
+
+    if entity_type == 'file':
+        lexer = get_lexer(language)
         if not language:
-            language, lexer = guess_language(file_name)
+            language, lexer = guess_language(file_name, local_file)
+        parser = DependencyParser(local_file or file_name, lexer)
+        stats.update({
+            'language': use_root_language(language, lexer),
+            'dependencies': parser.parse(),
+            'lines': number_lines_in_file(local_file or file_name),
+        })
 
-        language = use_root_language(language, lexer)
-
-        parser = DependencyParser(file_name, lexer)
-        dependencies = parser.parse()
-
-        stats = {
-            'language': language,
-            'dependencies': dependencies,
-            'lines': number_lines_in_file(file_name),
-            'lineno': lineno,
-            'cursorpos': cursorpos,
-        }
     return stats
 
 
-def guess_language(file_name):
+def guess_language(file_name, local_file):
     """Guess lexer and language for a file.
 
     Returns a tuple of (language_str, lexer_obj).
@@ -81,14 +78,14 @@ def guess_language(file_name):
     if language:
         lexer = get_lexer(language)
     else:
-        lexer = smart_guess_lexer(file_name)
+        lexer = smart_guess_lexer(file_name, local_file)
         if lexer:
             language = u(lexer.name)
 
     return language, lexer
 
 
-def smart_guess_lexer(file_name):
+def smart_guess_lexer(file_name, local_file):
     """Guess Pygments lexer for a file.
 
     Looks for a vim modeline in file contents, then compares the accuracy
@@ -99,7 +96,7 @@ def smart_guess_lexer(file_name):
 
     text = get_file_head(file_name)
 
-    lexer1, accuracy1 = guess_lexer_using_filename(file_name, text)
+    lexer1, accuracy1 = guess_lexer_using_filename(local_file or file_name, text)
     lexer2, accuracy2 = guess_lexer_using_modeline(text)
 
     if lexer1:
@@ -222,22 +219,21 @@ def number_lines_in_file(file_name):
 def standardize_language(language, plugin):
     """Maps a string to the equivalent Pygments language.
 
-    Returns a tuple of (language_str, lexer_obj).
+    Returns the standardized language string.
     """
 
     if not language:
-        return None, None
+        return None
 
     # standardize language for this plugin
     if plugin:
         plugin = plugin.split(' ')[-1].split('/')[0].split('-')[0]
         standardized = get_language_from_json(language, plugin)
         if standardized is not None:
-            return standardized, get_lexer(standardized)
+            return standardized
 
     # standardize language against default languages
-    standardized = get_language_from_json(language, 'default')
-    return standardized, get_lexer(standardized)
+    return get_language_from_json(language, 'default')
 
 
 def get_lexer(language):