v3.0.1

AUTHORS

@@ -0,0 +1,15 @@
+WakaTime is written and maintained by Alan Hamlett and
+various contributors:
+
+
+Development Lead
+----------------
+
+- Alan Hamlett <alan.hamlett@gmail.com>
+
+
+Patches and Suggestions
+-----------------------
+
+- Jimmy Selgen Nielsen <jimmy.selgen@gmail.com>
+- Patrik Kernstock <info@pkern.at>

HISTORY.rst

@@ -3,6 +3,41 @@ History
 -------
 
 
+3.0.1 (2014-12-23)
+++++++++++++++++++
+
+- parse use namespaces from php files
+
+
+3.0.0 (2014-12-23)
+++++++++++++++++++
+
+- upgrade external wakatime package to v3.0.1
+- detect libraries and frameworks for C++, Java, .NET, PHP, and Python files
+
+
+2.0.21 (2014-12-22)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.1.11
+- fix bug in offline logging when no response from api
+
+
+2.0.20 (2014-12-05)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.1.9
+- fix bug preventing offline heartbeats from being purged after uploaded
+
+
+2.0.19 (2014-12-04)
+++++++++++++++++++
+
+- upgrade external wakatime package to v2.1.8
+- fix UnicodeDecodeError when building user agent string
+- handle case where response is None
+
+
 2.0.18 (2014-11-30)
 ++++++++++++++++++
 

LICENSE

@@ -1,4 +1,6 @@
-Copyright (c) 2014 Alan Hamlett https://wakatime.com
+BSD 3-Clause License
+
+Copyright (c) 2014 by the respective authors (see AUTHORS file).
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -12,7 +14,7 @@ modification, are permitted provided that the following conditions are met:
   in the documentation and/or other materials provided
   with the distribution.
 
-* Neither the names of Wakatime or WakaTime, nor the names of its
+* Neither the names of WakaTime, nor the names of its
   contributors may be used to endorse or promote products derived
   from this software without specific prior written permission.
 

WakaTime.py

@@ -6,7 +6,7 @@ License:     BSD, see LICENSE for more details.
 Website:     https://wakatime.com/
 ==========================================================="""
 
-__version__ = '2.0.18'
+__version__ = '3.0.1'
 
 import sublime
 import sublime_plugin

packages/wakatime/HISTORY.rst

@@ -3,6 +3,43 @@ History
 -------
 
 
+3.0.1 (2014-12-23)
+++++++++++++++++++
+
+- handle unknown language when parsing dependencies
+
+
+3.0.0 (2014-12-23)
+++++++++++++++++++
+
+- detect libraries and frameworks for C++, Java, .NET, PHP, and Python files
+
+
+2.1.11 (2014-12-22)
++++++++++++++++++++
+
+- fix offline logging when response from api is None
+
+
+2.1.10 (2014-12-15)
++++++++++++++++++++
+
+- prevent queuing offline heartbeats which will never be valid (400 errors)
+
+
+2.1.9 (2014-12-05)
+++++++++++++++++++
+
+- fix bug preventing offline heartbeats from being purged after uploaded
+
+
+2.1.8 (2014-12-04)
+++++++++++++++++++
+
+- fix UnicodeDecodeError when building user agent string
+- handle case where response is None
+
+
 2.1.7 (2014-11-30)
 ++++++++++++++++++
 

packages/wakatime/LICENSE

@@ -1,4 +1,6 @@
-Copyright (c) 2013 Alan Hamlett https://wakatime.com
+BSD 3-Clause License
+
+Copyright (c) 2014 by the respective authors (see AUTHORS file).
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -12,7 +14,7 @@ modification, are permitted provided that the following conditions are met:
   in the documentation and/or other materials provided
   with the distribution.
 
-* Neither the names of Wakatime or WakaTime, nor the names of its
+* Neither the names of WakaTime, nor the names of its
   contributors may be used to endorse or promote products derived
   from this software without specific prior written permission.
 

packages/wakatime/wakatime/__init__.py

@@ -13,7 +13,7 @@
 from __future__ import print_function
 
 __title__ = 'wakatime'
-__version__ = '2.1.7'
+__version__ = '3.0.1'
 __author__ = 'Alan Hamlett'
 __license__ = 'BSD'
 __copyright__ = 'Copyright 2014 Alan Hamlett'
@@ -40,7 +40,7 @@ except ImportError:
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'packages'))
 
-from .compat import u, open, is_py2, is_py3
+from .compat import u, open, is_py3
 from .queue import Queue
 from .log import setup_logging
 from .project import find_project
@@ -122,7 +122,7 @@ def parseConfigFile(configFile):
                 print(traceback.format_exc())
                 return None
     except IOError:
-        print(u('Error: Could not read from config file {0}').format(configFile))
+        print(u('Error: Could not read from config file {0}').format(u(configFile)))
     return configs
 
 
@@ -142,14 +142,14 @@ def parseArguments(argv):
             description='Common interface for the WakaTime api.')
     parser.add_argument('--file', dest='targetFile', metavar='file',
             action=FileAction, required=True,
-            help='absolute path to file for current action')
+            help='absolute path to file for current heartbeat')
     parser.add_argument('--time', dest='timestamp', metavar='time',
             type=float,
             help='optional floating-point unix epoch timestamp; '+
                 'uses current time by default')
     parser.add_argument('--write', dest='isWrite',
             action='store_true',
-            help='note action was triggered from writing to a file')
+            help='note heartbeat was triggered from writing to a file')
     parser.add_argument('--plugin', dest='plugin',
             help='optional text editor plugin name and version '+
                 'for User-Agent header')
@@ -227,8 +227,8 @@ def should_ignore(fileName, patterns):
                     return pattern
             except re.error as ex:
                 log.warning(u('Regex error ({msg}) for ignore pattern: {pattern}').format(
-                    msg=str(ex),
-                    pattern=pattern,
+                    msg=u(ex),
+                    pattern=u(pattern),
                 ))
     except TypeError:
         pass
@@ -239,22 +239,22 @@ def get_user_agent(plugin):
     ver = sys.version_info
     python_version = '%d.%d.%d.%s.%d' % (ver[0], ver[1], ver[2], ver[3], ver[4])
     user_agent = u('wakatime/{ver} ({platform}) Python{py_ver}').format(
-        ver=__version__,
-        platform=platform.platform(),
+        ver=u(__version__),
+        platform=u(platform.platform()),
         py_ver=python_version,
     )
     if plugin:
         user_agent = u('{user_agent} {plugin}').format(
             user_agent=user_agent,
-            plugin=plugin,
+            plugin=u(plugin),
         )
     return user_agent
 
 
-def send_action(project=None, branch=None, stats=None, key=None, targetFile=None,
+def send_heartbeat(project=None, branch=None, stats={}, key=None, targetFile=None,
         timestamp=None, isWrite=None, plugin=None, offline=None,
         hidefilenames=None, **kwargs):
-    url = 'https://wakatime.com/api/v1/actions'
+    url = 'https://wakatime.com/api/v1/heartbeats'
     log.debug('Sending heartbeat to api at %s' % url)
     data = {
         'time': timestamp,
@@ -263,13 +263,15 @@ def send_action(project=None, branch=None, stats=None, key=None, targetFile=None
     if hidefilenames and targetFile is not None:
         data['file'] = data['file'].rsplit('/', 1)[-1].rsplit('\\', 1)[-1]
         if len(data['file'].strip('.').split('.', 1)) > 1:
-            data['file'] = u('HIDDEN.{ext}').format(ext=data['file'].strip('.').rsplit('.', 1)[-1])
+            data['file'] = u('HIDDEN.{ext}').format(ext=u(data['file'].strip('.').rsplit('.', 1)[-1]))
         else:
             data['file'] = u('HIDDEN')
     if stats.get('lines'):
         data['lines'] = stats['lines']
     if stats.get('language'):
         data['language'] = stats['language']
+    if stats.get('dependencies'):
+        data['dependencies'] = stats['dependencies']
     if isWrite:
         data['is_write'] = isWrite
     if project:
@@ -310,11 +312,12 @@ def send_action(project=None, branch=None, stats=None, key=None, targetFile=None
         if log.isEnabledFor(logging.DEBUG):
             exception_data['traceback'] = traceback.format_exc()
         if offline:
-            queue = Queue()
-            queue.push(data, plugin)
+            if response is None or response.getcode() != 400:
+                queue = Queue()
+                queue.push(data, json.dumps(stats), plugin)
             if log.isEnabledFor(logging.DEBUG):
                 log.warn(exception_data)
-            if response.getcode() in ALWAYS_LOG_CODES:
+            if response is not None and response.getcode() in ALWAYS_LOG_CODES:
                 log.error({
                     'response_code': response.getcode(),
                 })
@@ -327,41 +330,45 @@ def send_action(project=None, branch=None, stats=None, key=None, targetFile=None
         if log.isEnabledFor(logging.DEBUG):
             exception_data['traceback'] = traceback.format_exc()
         if offline:
-            queue = Queue()
-            queue.push(data, plugin)
+            if response is None or response.getcode() != 400:
+                queue = Queue()
+                queue.push(data, json.dumps(stats), plugin)
             if 'unknown url type: https' in u(sys.exc_info()[1]):
                 log.error(exception_data)
             elif log.isEnabledFor(logging.DEBUG):
                 log.warn(exception_data)
-            if response.getcode() in ALWAYS_LOG_CODES:
+            if response is not None and response.getcode() in ALWAYS_LOG_CODES:
                 log.error({
                     'response_code': response.getcode(),
                 })
         else:
             log.error(exception_data)
     else:
-        if response.getcode() == 201:
+        if response is not None and response.getcode() == 201:
             log.debug({
                 'response_code': response.getcode(),
             })
             return True
+        response_code = response.getcode() if response is not None else None
+        response_content = response.read() if response is not None else None
         if offline:
-            queue = Queue()
-            queue.push(data, plugin)
+            if response is None or response.getcode() != 400:
+                queue = Queue()
+                queue.push(data, json.dumps(stats), plugin)
             if log.isEnabledFor(logging.DEBUG):
                 log.warn({
-                    'response_code': response.getcode(),
-                    'response_content': response.read(),
+                    'response_code': response_code,
+                    'response_content': response_content,
                 })
             else:
                 log.error({
-                    'response_code': response.getcode(),
-                    'response_content': response.read(),
+                    'response_code': response_code,
+                    'response_content': response_content,
                 })
         else:
             log.error({
-                'response_code': response.getcode(),
-                'response_content': response.read(),
+                'response_code': response_code,
+                'response_content': response_content,
             })
     return False
 
@@ -379,7 +386,7 @@ def main(argv=None):
     ignore = should_ignore(args.targetFile, args.ignore)
     if ignore is not False:
         log.debug(u('File ignored because matches pattern: {pattern}').format(
-            pattern=ignore,
+            pattern=u(ignore),
         ))
         return 0
 
@@ -394,7 +401,7 @@ def main(argv=None):
             branch = project.branch()
             project_name = project.name()
 
-        if send_action(
+        if send_heartbeat(
                 project=project_name,
                 branch=branch,
                 stats=stats,
@@ -402,16 +409,17 @@ def main(argv=None):
             ):
             queue = Queue()
             while True:
-                action = queue.pop()
-                if action is None:
+                heartbeat = queue.pop()
+                if heartbeat is None:
                     break
-                sent = send_action(project=action['project'],
-                                   targetFile=action['file'],
-                                   timestamp=action['time'],
-                                   branch=action['branch'],
-                                   stats={'language': action['language'], 'lines': action['lines']},
-                                   key=args.key, isWrite=action['is_write'],
-                                   plugin=action['plugin'],
+                sent = send_heartbeat(project=heartbeat['project'],
+                                   targetFile=heartbeat['file'],
+                                   timestamp=heartbeat['time'],
+                                   branch=heartbeat['branch'],
+                                   stats=json.loads(heartbeat['stats']),
+                                   key=args.key,
+                                   isWrite=heartbeat['is_write'],
+                                   plugin=heartbeat['plugin'],
                                    offline=args.offline,
                                    hidefilenames=args.hidefilenames)
                 if not sent:
@@ -421,5 +429,5 @@ def main(argv=None):
         return 102 # api error
 
     else:
-        log.debug('File does not exist; ignoring this action.')
+        log.debug('File does not exist; ignoring this heartbeat.')
         return 0

packages/wakatime/wakatime/compat.py

@@ -21,9 +21,13 @@ is_py3 = (sys.version_info[0] == 3)
 if is_py2:
 
     def u(text):
-        if isinstance(text, str):
+        try:
             return text.decode('utf-8')
-        return unicode(text)
+        except:
+            try:
+                return unicode(text)
+            except:
+                return text
     open = codecs.open
     basestring = basestring
 
@@ -36,3 +40,38 @@ elif is_py3:
         return str(text)
     open = open
     basestring = (str, bytes)
+
+try:
+    from importlib import import_module
+except ImportError:
+    def _resolve_name(name, package, level):
+        """Return the absolute name of the module to be imported."""
+        if not hasattr(package, 'rindex'):
+            raise ValueError("'package' not set to a string")
+        dot = len(package)
+        for x in xrange(level, 1, -1):
+            try:
+                dot = package.rindex('.', 0, dot)
+            except ValueError:
+                raise ValueError("attempted relative import beyond top-level "
+                                 "package")
+        return "%s.%s" % (package[:dot], name)
+
+    def import_module(name, package=None):
+        """Import a module.
+        The 'package' argument is required when performing a relative import.
+        It specifies the package to use as the anchor point from which to
+        resolve the relative import to an absolute import.
+        """
+        if name.startswith('.'):
+            if not package:
+                raise TypeError("relative imports require the 'package' "
+                                + "argument")
+            level = 0
+            for character in name:
+                if character != '.':
+                    break
+                level += 1
+            name = _resolve_name(name[level:], package, level)
+        __import__(name)
+        return sys.modules[name]

packages/wakatime/wakatime/languages/__init__.py

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.languages
+    ~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from a source code file.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+import logging
+
+from ..compat import open, import_module
+
+
+log = logging.getLogger('WakaTime')
+
+
+class TokenParser(object):
+    """The base class for all dependency parsers. To add support for your
+    language, inherit from this class and implement the :meth:`parse` method
+    to return a list of dependency strings.
+    """
+    source_file = None
+    lexer = None
+    dependencies = []
+    tokens = []
+
+    def __init__(self, source_file, lexer=None):
+        self.source_file = source_file
+        self.lexer = lexer
+
+    def parse(self, tokens=[]):
+        """ Should return a list of dependencies.
+        """
+        if not tokens and not self.tokens:
+            self.tokens = self._extract_tokens()
+        raise Exception('Not yet implemented.')
+
+    def append(self, dep, truncate=True):
+        self._save_dependency(dep, truncate=truncate)
+
+    def _extract_tokens(self):
+        if self.lexer:
+            with open(self.source_file, 'r', encoding='utf-8') as fh:
+                return self.lexer.get_tokens_unprocessed(fh.read(512000))
+        return []
+
+    def _save_dependency(self, dep, truncate=True):
+        dep = dep.strip().split('.')[0].strip() if truncate else dep.strip()
+        if dep:
+            self.dependencies.append(dep)
+
+
+class DependencyParser(object):
+    source_file = None
+    lexer = None
+    parser = None
+
+    def __init__(self, source_file, lexer):
+        self.source_file = source_file
+        self.lexer = lexer
+
+        if self.lexer:
+            try:
+                module_name = self.lexer.__module__.split('.')[-1]
+                class_name = self.lexer.__class__.__name__.replace('Lexer', 'Parser', 1)
+                module = import_module('.%s' % module_name, package=__package__)
+                self.parser = getattr(module, class_name)
+            except ImportError as ex:
+                log.debug(ex)
+
+    def parse(self):
+        if self.parser:
+            plugin = self.parser(self.source_file, lexer=self.lexer)
+            dependencies = plugin.parse()
+            return list(set(dependencies))
+        return []

packages/wakatime/wakatime/languages/c_cpp.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.languages.c_cpp
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from C++ code.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+from ..compat import u
+
+
+class CppParser(TokenParser):
+
+    def parse(self, tokens=[]):
+        if not tokens and not self.tokens:
+            self.tokens = self._extract_tokens()
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if u(token).split('.')[-1] == 'Preproc':
+            self._process_preproc(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_preproc(self, token, content):
+        if content.strip().startswith('include ') or content.strip().startswith("include\t"):
+            content = content.replace('include', '', 1).strip()
+            self.append(content, truncate=False)
+
+    def _process_other(self, token, content):
+        pass

packages/wakatime/wakatime/languages/dotnet.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.languages.dotnet
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from .NET code.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+from ..compat import u
+
+
+class CSharpParser(TokenParser):
+
+    def parse(self, tokens=[]):
+        if not tokens and not self.tokens:
+            self.tokens = self._extract_tokens()
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if u(token).split('.')[-1] == 'Namespace':
+            self._process_namespace(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_namespace(self, token, content):
+        if content != 'import' and content != 'package' and content != 'namespace':
+            content = content.split('.')
+            content = content[0] if len(content) == 1 else '.'.join(content[0:len(content)-1])
+            self.append(content, truncate=False)
+
+    def _process_other(self, token, content):
+        pass

packages/wakatime/wakatime/languages/jvm.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.languages.java
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Java code.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+from ..compat import u
+
+
+class JavaParser(TokenParser):
+
+    def parse(self, tokens=[]):
+        if not tokens and not self.tokens:
+            self.tokens = self._extract_tokens()
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if u(token).split('.')[-1] == 'Namespace':
+            self._process_namespace(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_namespace(self, token, content):
+        if content != 'import' and content != 'package' and content != 'namespace':
+            content = content.split('.')
+            content = content[0] if len(content) == 1 else '.'.join(content[0:len(content)-1])
+            self.append(content, truncate=False)
+
+    def _process_other(self, token, content):
+        pass

packages/wakatime/wakatime/languages/php.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.languages.php
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from PHP code.
+
+    :copyright: (c) 2014 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+from ..compat import u
+
+
+class PhpParser(TokenParser):
+    state = None
+    parens = 0
+
+    def parse(self, tokens=[]):
+        if not tokens and not self.tokens:
+            self.tokens = self._extract_tokens()
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if u(token).split('.')[-1] == 'Keyword':
+            self._process_keyword(token, content)
+        elif u(token) == 'Token.Literal.String.Single' or u(token) == 'Token.Literal.String.Double':
+            self._process_literal_string(token, content)
+        elif u(token) == 'Token.Name.Other':
+            self._process_name(token, content)
+        elif u(token) == 'Token.Name.Function':
+            self._process_function(token, content)
+        elif u(token).split('.')[-1] == 'Punctuation':
+            self._process_punctuation(token, content)
+        elif u(token).split('.')[-1] == 'Text':
+            self._process_text(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_name(self, token, content):
+        if self.state == 'use':
+            content = content.split("\\")
+            content = content[0] if len(content) == 1 else "\\".join(content[0:len(content)-1])
+            self.append(content, truncate=False)
+
+    def _process_function(self, token, content):
+        if self.state == 'use function':
+            content = content.split("\\")
+            content = content[0] if len(content) == 1 else "\\".join(content[0:len(content)-1])
+            self.append(content, truncate=False)
+            self.state = 'use'
+
+    def _process_keyword(self, token, content):
+        if content == 'include' or content == 'include_once' or content == 'require' or content == 'require_once':
+            self.state = 'include'
+        elif content == 'use':
+            self.state = 'use'
+        elif content == 'as':
+            self.state = 'as'
+        elif self.state == 'use' and content == 'function':
+            self.state = 'use function'
+        else:
+            self.state = None
+
+    def _process_literal_string(self, token, content):
+        if self.state == 'include':
+            if content != '"':
+                content = content.strip()
+                if u(token) == 'Token.Literal.String.Double':
+                    content = u('"{0}"').format(content)
+                self.append(content, truncate=False)
+                self.state = None
+
+    def _process_punctuation(self, token, content):
+        if content == '(':
+            self.parens += 1
+        elif content == ')':
+            self.parens -= 1
+        elif (self.state == 'use' or self.state == 'as') and content == ',':
+            self.state = 'use'
+        else:
+            self.state = None
+
+    def _process_text(self, token, content):
+        pass
+
+    def _process_other(self, token, content):
+        self.state = None

packages/wakatime/wakatime/languages/python.py

@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+"""
+    wakatime.languages.python
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Parse dependencies from Python code.
+
+    :copyright: (c) 2013 Alan Hamlett.
+    :license: BSD, see LICENSE for more details.
+"""
+
+from . import TokenParser
+from ..compat import u
+
+
+class PythonParser(TokenParser):
+    state = None
+    parens = 0
+    nonpackage = False
+
+    def parse(self, tokens=[]):
+        if not tokens and not self.tokens:
+            self.tokens = self._extract_tokens()
+        for index, token, content in self.tokens:
+            self._process_token(token, content)
+        return self.dependencies
+
+    def _process_token(self, token, content):
+        if u(token).split('.')[-1] == 'Namespace':
+            self._process_namespace(token, content)
+        elif u(token).split('.')[-1] == 'Name':
+            self._process_name(token, content)
+        elif u(token).split('.')[-1] == 'Word':
+            self._process_word(token, content)
+        elif u(token).split('.')[-1] == 'Operator':
+            self._process_operator(token, content)
+        elif u(token).split('.')[-1] == 'Punctuation':
+            self._process_punctuation(token, content)
+        elif u(token).split('.')[-1] == 'Text':
+            self._process_text(token, content)
+        else:
+            self._process_other(token, content)
+
+    def _process_namespace(self, token, content):
+        if self.state is None:
+            self.state = content
+        else:
+            self._process_import(token, content)
+
+    def _process_name(self, token, content):
+        if self.state is not None:
+            if self.nonpackage:
+                self.nonpackage = False
+            else:
+                if self.state == 'from':
+                    self.append(content)
+                if self.state == 'from-2' and content != 'import':
+                    self.append(content)
+                elif self.state == 'import':
+                    self.append(content)
+                elif self.state == 'import-2':
+                    self.append(content)
+                else:
+                    self.state = None
+
+    def _process_word(self, token, content):
+        if self.state is not None:
+            if self.nonpackage:
+                self.nonpackage = False
+            else:
+                if self.state == 'from':
+                    self.append(content)
+                if self.state == 'from-2' and content != 'import':
+                    self.append(content)
+                elif self.state == 'import':
+                    self.append(content)
+                elif self.state == 'import-2':
+                    self.append(content)
+                else:
+                    self.state = None
+
+    def _process_operator(self, token, content):
+        if self.state is not None:
+            if content == '.':
+                self.nonpackage = True
+
+    def _process_punctuation(self, token, content):
+        if content == '(':
+            self.parens += 1
+        elif content == ')':
+            self.parens -= 1
+        self.nonpackage = False
+
+    def _process_text(self, token, content):
+        if self.state is not None:
+            if content == "\n" and self.parens == 0:
+                self.state = None
+                self.nonpackage = False
+
+    def _process_other(self, token, content):
+        pass
+
+    def _process_import(self, token, content):
+        self.nonpackage = False
+        if self.state == 'from':
+            self.append(content)
+            self.state = 'from-2'
+        elif self.state == 'from-2' and content != 'import':
+            self.append(content)
+        elif self.state == 'import':
+            self.append(content)
+            self.state = 'import-2'
+        elif self.state == 'import-2':
+            self.append(content)
+        else:
+            self.state = None

packages/wakatime/wakatime/queue.py

@@ -15,6 +15,7 @@ import logging
 import os
 import traceback
 from time import sleep
+
 try:
     import sqlite3
     HAS_SQL = True
@@ -31,35 +32,35 @@ class Queue(object):
     def connect(self):
         conn = sqlite3.connect(self.DB_FILE)
         c = conn.cursor()
-        c.execute('''CREATE TABLE IF NOT EXISTS action (
+        c.execute('''CREATE TABLE IF NOT EXISTS heartbeat (
             file text,
             time real,
             project text,
-            language text,
-            lines integer,
             branch text,
             is_write integer,
+            stats text,
+            misc text,
             plugin text)
         ''')
         return (conn, c)
 
 
-    def push(self, data, plugin):
+    def push(self, data, stats, plugin, misc=None):
         if not HAS_SQL:
             return
         try:
             conn, c = self.connect()
-            action = {
+            heartbeat = {
                 'file': data.get('file'),
                 'time': data.get('time'),
                 'project': data.get('project'),
-                'language': data.get('language'),
-                'lines': data.get('lines'),
                 'branch': data.get('branch'),
                 'is_write': 1 if data.get('is_write') else 0,
+                'stats': stats,
+                'misc': misc,
                 'plugin': plugin,
             }
-            c.execute('INSERT INTO action VALUES (:file,:time,:project,:language,:lines,:branch,:is_write,:plugin)', action)
+            c.execute('INSERT INTO heartbeat VALUES (:file,:time,:project,:branch,:is_write,:stats,:misc,:plugin)', heartbeat)
             conn.commit()
             conn.close()
         except sqlite3.Error:
@@ -71,7 +72,7 @@ class Queue(object):
             return None
         tries = 3
         wait = 0.1
-        action = None
+        heartbeat = None
         try:
             conn, c = self.connect()
         except sqlite3.Error:
@@ -81,13 +82,13 @@ class Queue(object):
         while loop and tries > -1:
             try:
                 c.execute('BEGIN IMMEDIATE')
-                c.execute('SELECT * FROM action LIMIT 1')
+                c.execute('SELECT * FROM heartbeat LIMIT 1')
                 row = c.fetchone()
                 if row is not None:
                     values = []
                     clauses = []
                     index = 0
-                    for row_name in ['file', 'time', 'project', 'language', 'lines', 'branch', 'is_write']:
+                    for row_name in ['file', 'time', 'project', 'branch', 'is_write']:
                         if row[index] is not None:
                             clauses.append('{0}=?'.format(row_name))
                             values.append(row[index])
@@ -95,19 +96,19 @@ class Queue(object):
                             clauses.append('{0} IS NULL'.format(row_name))
                         index += 1
                     if len(values) > 0:
-                        c.execute('DELETE FROM action WHERE {0}'.format(' AND '.join(clauses)), values)
+                        c.execute('DELETE FROM heartbeat WHERE {0}'.format(' AND '.join(clauses)), values)
                     else:
-                        c.execute('DELETE FROM action WHERE {0}'.format(' AND '.join(clauses)))
+                        c.execute('DELETE FROM heartbeat WHERE {0}'.format(' AND '.join(clauses)))
                 conn.commit()
                 if row is not None:
-                    action = {
+                    heartbeat = {
                         'file': row[0],
                         'time': row[1],
                         'project': row[2],
-                        'language': row[3],
-                        'lines': row[4],
-                        'branch': row[5],
-                        'is_write': True if row[6] is 1 else False,
+                        'branch': row[3],
+                        'is_write': True if row[4] is 1 else False,
+                        'stats': row[5],
+                        'misc': row[6],
                         'plugin': row[7],
                     }
                 loop = False
@@ -119,4 +120,4 @@ class Queue(object):
             conn.close()
         except sqlite3.Error:
             log.debug(traceback.format_exc())
-        return action
+        return heartbeat

packages/wakatime/wakatime/stats.py

@@ -14,6 +14,7 @@ import os
 import sys
 
 from .compat import u, open
+from .languages import DependencyParser
 
 if sys.version_info[0] == 2:
     sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'packages', 'pygments_py2'))
@@ -46,20 +47,17 @@ TRANSLATIONS = {
 
 
 def guess_language(file_name):
-    if file_name:
-        language = guess_language_from_extension(file_name.rsplit('.', 1)[-1])
-        if language:
-            return language
-    lexer = None
+    language, lexer = None, None
     try:
         with open(file_name, 'r', encoding='utf-8') as fh:
             lexer = guess_lexer_for_filename(file_name, fh.read(512000))
     except:
         pass
-    if lexer:
-        return translate_language(u(lexer.name))
-    else:
-        return None
+    if file_name:
+        language = guess_language_from_extension(file_name.rsplit('.', 1)[-1])
+    if lexer and language is None:
+        language = translate_language(u(lexer.name))
+    return language, lexer
 
 
 def guess_language_from_extension(extension):
@@ -89,8 +87,12 @@ def number_lines_in_file(file_name):
 
 
 def get_file_stats(file_name):
+    language, lexer = guess_language(file_name)
+    parser = DependencyParser(file_name, lexer)
+    dependencies = parser.parse()
     stats = {
-        'language': guess_language(file_name),
+        'language': language,
+        'dependencies': dependencies,
         'lines': number_lines_in_file(file_name),
     }
     return stats