Introduce Python plugin

.gitlab/build-codecompass.sh

@@ -41,13 +41,15 @@ export LDFLAGS="-Wl,-rpath-link,$DEPS_INSTALL_RUNTIME_DIR/openssl-install/lib "\
 "-Wl,-rpath-link,$DEPS_INSTALL_RUNTIME_DIR/postgresql-install/lib"
 
 export LD_LIBRARY_PATH=$DEPS_INSTALL_RUNTIME_DIR/odb-install/lib\
+:$DEPS_INSTALL_RUNTIME_DIR/python-install/lib\
 :$LD_LIBRARY_PATH
 
 export CMAKE_PREFIX_PATH=$DEPS_INSTALL_RUNTIME_DIR/libgit2-install\
 :$DEPS_INSTALL_RUNTIME_DIR/odb-install\
 :$DEPS_INSTALL_RUNTIME_DIR/thrift-install\
 :$DEPS_INSTALL_RUNTIME_DIR/boost-install\
 :$DEPS_INSTALL_RUNTIME_DIR/llvm-install\
+:$DEPS_INSTALL_RUNTIME_DIR/python-install\
 :$DEPS_INSTALL_RUNTIME_DIR/openldap-install
 
 export GTEST_ROOT=$DEPS_INSTALL_BUILD_DIR/gtest-install

.gitlab/build-deps.sh

@@ -240,11 +240,14 @@ export LD_LIBRARY_PATH=$DEPS_INSTALL_RUNTIME_DIR/openssl-install/lib:$LD_LIBRARY
   --quiet \
   --prefix=$DEPS_INSTALL_RUNTIME_DIR/python-install \
   --with-openssl=$DEPS_INSTALL_RUNTIME_DIR/openssl-install \
+  --enable-shared \
   --enable-optimizations
 make install --quiet --jobs $(nproc)
 rm -f $PACKAGES_DIR/Python-3.9.0.tar.xz
 
 export PATH=$DEPS_INSTALL_RUNTIME_DIR/python-install/bin:$PATH
+export LD_LIBRARY_PATH=$DEPS_INSTALL_RUNTIME_DIR/python-install/lib:$LD_LIBRARY_PATH
+# --enabled-shared: build libpython.so instead of libpython.a, must be on LD_LIBRARY_PATH
 
 ##############
 # LLVM/Clang #
@@ -313,7 +316,7 @@ if [ ! -f $DEPS_INSTALL_RUNTIME_DIR/boost-install/lib/libboost_program_options.s
 
   ./bootstrap.sh \
     --prefix=$DEPS_INSTALL_RUNTIME_DIR/boost-install \
-    --with-python=$DEPS_INSTALL_RUNTIME_DIR/python-install/bin/python
+    --with-python=$DEPS_INSTALL_RUNTIME_DIR/python-install/bin/python3
   ./b2 -j $(nproc) install
 
   rm -f $PACKAGES_DIR/boost_1_74_0.tar.gz

.gitlab/cc-env.sh

@@ -8,6 +8,7 @@ DEPS_INSTALL_DIR=$ROOT_DIR/deps-runtime
 CC_INSTALL_DIR=$ROOT_DIR/cc-install
 
 export LD_LIBRARY_PATH=$DEPS_INSTALL_DIR/libgit2-install/lib64\
+:$DEPS_INSTALL_DIR/libgit2-install/lib\
 :$DEPS_INSTALL_DIR/openssl-install/lib\
 :$DEPS_INSTALL_DIR/graphviz-install/lib\
 :$DEPS_INSTALL_DIR/libmagic-install/lib\
@@ -17,8 +18,10 @@ export LD_LIBRARY_PATH=$DEPS_INSTALL_DIR/libgit2-install/lib64\
 :$DEPS_INSTALL_DIR/llvm-install/lib\
 :$DEPS_INSTALL_DIR/libtool-install/lib\
 :$DEPS_INSTALL_DIR/postgresql-install/lib\
+:$DEPS_INSTALL_DIR/python-install/lib\
 :$DEPS_INSTALL_DIR/openldap-install/lib\
 :$LD_LIBRARY_PATH
+# Note: libgit2-install/lib required on Ubuntu; libgit2-install/lib64 on SUSE
 
 export PATH=$DEPS_INSTALL_DIR/jdk-install/bin\
 :$DEPS_INSTALL_DIR/ctags-install/bin\
@@ -30,6 +33,7 @@ export PATH=$DEPS_INSTALL_DIR/jdk-install/bin\
 
 export MAGIC=$DEPS_INSTALL_DIR/libmagic-install/share/misc/magic.mgc
 export JAVA_HOME=$DEPS_INSTALL_DIR/jdk-install
+export PYTHONHOME=$DEPS_INSTALL_DIR/python-install
 
 if [ -f $ROOT_DIR/ccdb-tool/venv/bin/activate ]; then
   source $ROOT_DIR/ccdb-tool/venv/bin/activate

.gitlab/ci.yml

@@ -62,7 +62,7 @@ tarball suse-15:
     extends: .tarball
     image: opensuse/leap:15
     cache:
-        key: "leap"
+        key: "leap-$CI_COMMIT_REF_SLUG"
     variables:
         GCC_VERSION: 9.3.0
         ODB_VERSION: 2.5.0
@@ -85,7 +85,7 @@ tarball suse-42.1:
     extends: .tarball
     image: opensuse/archive:42.1
     cache:
-        key: "malachite"
+        key: "malachite-$CI_COMMIT_REF_SLUG"
     variables:
         GCC_VERSION: 5.5.0
         ODB_VERSION: 2.4.0
@@ -111,7 +111,7 @@ tarball ubuntu-16.04:
     extends: .tarball
     image: ubuntu:16.04
     cache:
-        key: "xenial"
+        key: "xenial-$CI_COMMIT_REF_SLUG"
     variables:
         GCC_VERSION: 5.5.0
         ODB_VERSION: 2.4.0

doc/deps.md

@@ -40,6 +40,7 @@ be installed from the official repository of the given Linux distribution.
 - **`libgtest-dev`**: For testing CodeCompass.
   ***See [Known issues](#known-issues)!***
 - **`libldap2-dev`**: For LDAP authentication.
+- **`python3.8`**: For python parsing.
 
 ## Quick guide
 
@@ -52,7 +53,7 @@ known issues.
 sudo apt install git cmake make g++ gcc-7-plugin-dev libboost-all-dev \
   llvm-10-dev clang-10 libclang-10-dev \
   default-jdk libssl1.0-dev libgraphviz-dev libmagic-dev libgit2-dev ctags doxygen \
-  libldap2-dev libgtest-dev npm
+  libldap2-dev libgtest-dev npm python3.8
 ```
 
 #### Ubuntu 20.04 ("Focal Fossa") LTS

model/include/model/buildaction.h

@@ -25,6 +25,7 @@ struct BuildAction
   {
     Compile,
     Link,
+    Interpret,
     Other
   };
 

plugins/cpp/webgui/js/cppInfoTree.js

@@ -291,6 +291,7 @@ function (model, viewHandler, util) {
   }
 
   var cppInfoTree = {
+    id: 'cpp-info-tree',
     render : function (elementInfo) {
       var ret = [];
 

plugins/python/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_subdirectory(parser)
+add_subdirectory(model)
+add_subdirectory(service)
+
+install_webplugin(webgui)

plugins/python/model/CMakeLists.txt

@@ -0,0 +1,17 @@
+set(ODB_SOURCES
+        include/model/pythonastnode.h
+        include/model/pythonclass.h
+        include/model/pythondocumentation.h
+        include/model/pythonentity.h
+        include/model/pythonfunction.h
+        include/model/pythonimport.h
+        include/model/pythoninheritance.h
+        include/model/pythontype.h
+        include/model/pythonvariable.h)
+
+generate_odb_files("${ODB_SOURCES}")
+
+add_odb_library(pythonmodel ${ODB_CXX_SOURCES})
+target_link_libraries(pythonmodel model)
+
+install_sql()

plugins/python/model/include/model/pythonastnode.h

@@ -0,0 +1,164 @@
+#ifndef CC_MODEL_PYTHONASTNODE_H
+#define CC_MODEL_PYTHONASTNODE_H
+
+#include <string>
+
+#include <odb/nullable.hxx>
+#include <odb/core.hxx>
+#include <odb/lazy-ptr.hxx>
+
+#include <model/fileloc.h>
+
+#include <util/hash.h>
+
+namespace cc
+{
+namespace model
+{
+
+typedef std::uint64_t PythonAstNodeId;
+
+#pragma db object
+struct PythonAstNode
+{
+    enum class SymbolType
+    {
+        Variable,
+        Function,
+        Class,
+        Module,
+        Other
+    };
+
+    enum class AstType
+    {
+        Declaration,
+        Usage,
+        Other
+    };
+
+    virtual ~PythonAstNode() {}
+
+    #pragma db id
+    PythonAstNodeId id = 0;
+
+    std::string astValue;
+
+    std::string qualifiedName;
+
+    #pragma db null
+    FileLoc location;
+
+    SymbolType symbolType = SymbolType::Other;
+
+    AstType astType = AstType::Other;
+
+    std::string toString() const;
+
+    bool operator< (const PythonAstNode& other) const { return id <  other.id; }
+    bool operator==(const PythonAstNode& other) const { return id == other.id; }
+};
+
+typedef std::shared_ptr<PythonAstNode> PythonAstNodePtr;
+
+inline std::string symbolTypeToString(PythonAstNode::SymbolType type_)
+{
+    switch (type_)
+    {
+        case PythonAstNode::SymbolType::Variable: return "Variable";
+        case PythonAstNode::SymbolType::Function: return "Function";
+        case PythonAstNode::SymbolType::Class: return "Class";
+        case PythonAstNode::SymbolType::Module: return "Module";
+        case PythonAstNode::SymbolType::Other: return "Other";
+    }
+
+    return std::string();
+}
+
+inline std::string astTypeToString(PythonAstNode::AstType type)
+{
+    switch (type) {
+        case PythonAstNode::AstType::Usage: return "Usage";
+        case PythonAstNode::AstType::Declaration: return "Declaration";
+        case PythonAstNode::AstType::Other: return "Other";
+    }
+
+    return std::string();
+}
+
+inline std::string PythonAstNode::toString() const
+{
+    return std::string("PythonAstNode")
+        .append("\nid = ").append(std::to_string(id))
+        .append("\nastValue = ").append(astValue)
+        .append("\nqualifiedName = ").append(qualifiedName)
+        .append("\nlocation = ").append(location.file->path).append(" (")
+        .append(std::to_string(
+                static_cast<signed>(location.range.start.line))).append(":")
+        .append(std::to_string(
+                static_cast<signed>(location.range.start.column))).append(" - ")
+        .append(std::to_string(
+                static_cast<signed>(location.range.end.line))).append(":")
+        .append(std::to_string(
+                static_cast<signed>(location.range.end.column))).append(")")
+        .append("\nsymbolType = ").append(symbolTypeToString(symbolType))
+        .append("\nastType = ").append(astTypeToString(astType));
+}
+
+inline std::uint64_t createIdentifier(const PythonAstNode& astNode_)
+{
+    using SymbolTypeInt
+    = std::underlying_type<model::PythonAstNode::SymbolType>::type;
+    using AstTypeInt
+    = std::underlying_type<model::PythonAstNode::AstType>::type;
+
+    std::string res;
+
+    res
+        .append(astNode_.astValue).append(":")
+        .append(astNode_.qualifiedName).append(":")
+        .append(std::to_string(static_cast<SymbolTypeInt>(astNode_.symbolType))).append(":")
+        .append(std::to_string(static_cast<AstTypeInt>(astNode_.astType))).append(":");
+
+    if (astNode_.location.file){
+        res
+            .append(std::to_string(astNode_.location.file->id)).append(":")
+            .append(std::to_string(astNode_.location.range.start.line)).append(":")
+            .append(std::to_string(astNode_.location.range.start.column)).append(":")
+            .append(std::to_string(astNode_.location.range.end.line)).append(":")
+            .append(std::to_string(astNode_.location.range.end.column)).append(":");
+    } else {
+        res.append("null");
+    }
+
+    return util::fnvHash(res);
+}
+
+inline std::uint64_t createAstNodeInfoEntityHash(const PythonAstNode& astNode_)
+{
+    return util::fnvHash(astNode_.qualifiedName);
+}
+
+#pragma db view \
+  object(PythonAstNode) object(File = LocFile : PythonAstNode::location.file) \
+  query ((?) + "GROUP BY" + LocFile::id + "ORDER BY" + LocFile::id)
+struct PythonAstCountGroupByFiles
+{
+    #pragma db column(LocFile::id)
+    FileId file;
+
+    #pragma db column("count(" + PythonAstNode::id + ")")
+    std::size_t count;
+};
+
+#pragma db view object(PythonAstNode)
+struct PythonAstCount
+{
+    #pragma db column("count(" + PythonAstNode::id + ")")
+    std::size_t count;
+};
+
+}
+}
+
+#endif

plugins/python/model/include/model/pythonclass.h

@@ -0,0 +1,77 @@
+#ifndef CC_MODEL_PYTHONCLASS_H
+#define CC_MODEL_PYTHONCLASS_H
+
+#include "pythonentity.h"
+
+namespace cc
+{
+namespace model
+{
+
+#pragma db object
+struct PythonClass : PythonEntity
+{
+    std::string toString() const
+    {
+        return std::string("PythonClass")
+                .append("\nid = ").append(std::to_string(id))
+                .append("\nqualifiedName = ").append(qualifiedName);
+    }
+};
+
+typedef std::shared_ptr<PythonClass> PythonClassPtr;
+
+#pragma db object
+struct PythonClassMember
+{
+    enum Kind
+    {
+        Attribute,
+        Method,
+        Class
+    };
+
+    #pragma db id auto
+    int id;
+
+    #pragma db unique
+    PythonAstNodeId astNodeId;
+
+    PythonEntityId memberId;
+    PythonEntityId classId;
+
+    Kind kind;
+    bool staticMember = false;
+
+    std::string toString() const
+    {
+        return std::string("PythonClassMember")
+                .append("\nid = ").append(std::to_string(id))
+                .append("\nmemberId = ").append(std::to_string(memberId))
+                .append("\nclassId = ").append(std::to_string(classId))
+                .append("\nstaticMember = ").append(std::to_string(staticMember))
+                .append("\nkind = ").append(kind == Attribute ? "Attribute" :
+                            kind == Method ? "Method" : "Class");
+    }
+};
+
+typedef std::shared_ptr<PythonClassMember> PythonClassMemberPtr;
+
+#pragma db view object(PythonClass)
+struct PythonClassCount
+{
+    #pragma db column("count(" + PythonClass::id + ")")
+    std::size_t count;
+};
+
+#pragma db view object(PythonClassMember)
+struct PythonClassMemberCount
+{
+    #pragma db column("count(" + PythonClassMember::id + ")")
+    std::size_t count;
+};
+
+}
+}
+
+#endif