Skip to content

Feature/uctags2 #2983

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Nov 29, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
import org.opengrok.indexer.analysis.archive.GZIPAnalyzerFactory;
import org.opengrok.indexer.analysis.archive.TarAnalyzerFactory;
import org.opengrok.indexer.analysis.archive.ZipAnalyzerFactory;
import org.opengrok.indexer.analysis.asm.AsmAnalyzerFactory;
import org.opengrok.indexer.analysis.c.CAnalyzerFactory;
import org.opengrok.indexer.analysis.c.CxxAnalyzerFactory;
import org.opengrok.indexer.analysis.clojure.ClojureAnalyzerFactory;
Expand Down Expand Up @@ -241,6 +242,7 @@ public class AnalyzerGuru {
private static final Map<String, Long> ANALYZER_VERSIONS = new HashMap<>();

private static final LangTreeMap langMap = new LangTreeMap();
private static final LangTreeMap defaultLangMap = new LangTreeMap();

/*
* If you write your own analyzer please register it here. The order is
Expand Down Expand Up @@ -297,7 +299,8 @@ public class AnalyzerGuru {
new RubyAnalyzerFactory(),
new EiffelAnalyzerFactory(),
new VerilogAnalyzerFactory(),
new TypeScriptAnalyzerFactory()
new TypeScriptAnalyzerFactory(),
new AsmAnalyzerFactory()
};

for (AnalyzerFactory analyzer : analyzers) {
Expand Down Expand Up @@ -331,15 +334,15 @@ public class AnalyzerGuru {
* {@link FileAnalyzerFactory} subclasses are revised to target more or
* different files.
* @return a value whose lower 32-bits are a static value
* 20191006_00
* 20191120_00
* for the current implementation and whose higher-32 bits are non-zero if
* {@link #addExtension(java.lang.String, AnalyzerFactory)}
* or
* {@link #addPrefix(java.lang.String, AnalyzerFactory)}
* has been called.
*/
public static long getVersionNo() {
final int ver32 = 20191006_00; // Edit comment above too!
final int ver32 = 20191120_00; // Edit comment above too!
long ver = ver32;
if (customizationHashCode != 0) {
ver |= (long) customizationHashCode << 32;
Expand Down Expand Up @@ -418,6 +421,25 @@ private static void registerAnalyzer(AnalyzerFactory factory) {
String fileTypeName = fa.getFileTypeName();
FILETYPE_FACTORIES.put(fileTypeName, factory);
ANALYZER_VERSIONS.put(fileTypeName, fa.getVersionNo());

// Possibly configure default LANG mappings for the factory.
String ctagsLang = factory.getAnalyzer().getCtagsLang();
if (ctagsLang != null) {
List<String> prefixes = factory.getPrefixes();
if (prefixes != null) {
for (String prefix : prefixes) {
defaultLangMap.add(prefix, ctagsLang);
}
}

List<String> suffixes = factory.getSuffixes();
if (suffixes != null) {
for (String suffix : suffixes) {
// LangMap needs a "." to signify a file extension.
defaultLangMap.add("." + suffix, ctagsLang);
}
}
}
}

/**
Expand Down Expand Up @@ -479,10 +501,11 @@ public static void addExtension(String extension, AnalyzerFactory factory) {
/**
* Gets an unmodifiable view of the language mappings resulting from
* {@link #addExtension(String, AnalyzerFactory)} and
* {@link #addPrefix(String, AnalyzerFactory)}.
* {@link #addPrefix(String, AnalyzerFactory)} merged with default language
* mappings of OpenGrok's analyzers.
*/
public static LangMap getLangMap() {
return langMap.unmodifiable();
return langMap.mergeSecondary(defaultLangMap).unmodifiable();
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@ public class Ctags implements Resettable {

private final RuntimeEnvironment env;
private volatile boolean closing;
private final LangTreeMap defaultLangMap = new LangTreeMap();
private LangMap langMap;
private List<String> command;
private Process ctags;
Expand All @@ -74,8 +73,13 @@ public class Ctags implements Resettable {

private boolean junit_testing = false;

/**
* Initializes an instance with the current
* {@link AnalyzerGuru#getLangMap()}.
*/
public Ctags() {
env = RuntimeEnvironment.getInstance();
langMap = AnalyzerGuru.getLangMap();
}

/**
Expand Down Expand Up @@ -144,6 +148,11 @@ public List<String> getArgv() {
}

private void initialize() {
/*
* Call the following principally to properly initialize when running
* JUnit tests. opengrok-indexer and opengrok-web call it too but
* validating its return code and logging (and possibly aborting).
*/
env.validateUniversalCtags();

command = new ArrayList<>();
Expand All @@ -166,15 +175,6 @@ private void initialize() {
command.add("--fields=-anf+iKnS");
command.add("--excmd=pattern");

defaultLangMap.clear();
defaultLangMap.add(".KSHLIB", "sh"); // RFE #17849. Upper-case file spec
defaultLangMap.add(".PLB", "sql"); // RFE #19208. Upper-case file spec
defaultLangMap.add(".PLS", "sql"); // RFE #19208. Upper-case file spec
defaultLangMap.add(".PLD", "sql"); // RFE #19208. Upper-case file spec
defaultLangMap.add(".PKS", "sql"); // RFE #19208 ? Upper-case file spec
defaultLangMap.add(".PKB", "sql"); // # 1763. Upper-case file spec
defaultLangMap.add(".PCK", "sql"); // # 1763. Upper-case file spec

//Ideally all below should be in ctags, or in outside config file,
//we might run out of command line SOON
//Also note, that below ctags definitions HAVE to be in POSIX
Expand All @@ -200,10 +200,10 @@ private void initialize() {

//PLEASE add new languages ONLY with POSIX syntax (see above wiki link)

if (langMap != null) {
command.addAll(langMap.mergeSecondary(defaultLangMap).getCtagsArgs());
if (langMap == null) {
LOGGER.warning("langMap property is null");
} else {
command.addAll(defaultLangMap.getCtagsArgs());
command.addAll(langMap.getCtagsArgs());
}

/* Add extra command line options for ctags. */
Expand Down Expand Up @@ -254,7 +254,6 @@ private void addRustSupport(List<String> command) {
if (!env.getCtagsLanguages().contains("Rust")) { // Built-in would be capitalized.
command.add("--langdef=rust"); // Lower-case if user-defined.
}
defaultLangMap.add(".RS", "rust"); // Upper-case file spec

// The following are not supported yet in Universal Ctags b13cb551
command.add("--regex-rust=/^[[:space:]]*(pub[[:space:]]+)?(static|const)[[:space:]]+(mut[[:space:]]+)?" +
Expand All @@ -270,8 +269,6 @@ private void addPowerShellSupport(List<String> command) {
if (!env.getCtagsLanguages().contains("PowerShell")) { // Built-in would be capitalized.
command.add("--langdef=powershell"); // Lower-case if user-defined.
}
defaultLangMap.add(".PS1", "powershell"); // Upper-case file spec
defaultLangMap.add(".PSM1", "powershell"); // Upper-case file spec

command.add("--regex-powershell=/\\$(\\{[^}]+\\})/\\1/v,variable/");
command.add("--regex-powershell=/\\$([[:alnum:]_]+([:.][[:alnum:]_]+)*)/\\1/v,variable/");
Expand All @@ -292,7 +289,6 @@ private void addPascalSupport(List<String> command) {
if (!env.getCtagsLanguages().contains("Pascal")) { // Built-in would be capitalized.
command.add("--langdef=pascal"); // Lower-case if user-defined.
}
defaultLangMap.add(".PAS", "pascal"); // Upper-case file spec

command.add("--regex-pascal=/([[:alnum:]_]+)[[:space:]]*=[[:space:]]*\\([[:space:]]*[[:alnum:]_][[:space:]]*\\)/\\1/t,Type/");
command.add("--regex-pascal=/([[:alnum:]_]+)[[:space:]]*=[[:space:]]*class[[:space:]]*[^;]*$/\\1/c,Class/");
Expand All @@ -310,7 +306,7 @@ private void addSwiftSupport(List<String> command) {
if (!env.getCtagsLanguages().contains("Swift")) { // Built-in would be capitalized.
command.add("--langdef=swift"); // Lower-case if user-defined.
}
defaultLangMap.add(".SWIFT", "swift"); // Upper-case file spec

command.add("--regex-swift=/enum[[:space:]]+([^\\{\\}]+).*$/\\1/n,enum,enums/");
command.add("--regex-swift=/typealias[[:space:]]+([^:=]+).*$/\\1/t,typealias,typealiases/");
command.add("--regex-swift=/protocol[[:space:]]+([^:\\{]+).*$/\\1/p,protocol,protocols/");
Expand All @@ -325,8 +321,6 @@ private void addKotlinSupport(List<String> command) {
if (!env.getCtagsLanguages().contains("Kotlin")) { // Built-in would be capitalized.
command.add("--langdef=kotlin"); // Lower-case if user-defined.
}
defaultLangMap.add(".KT", "kotlin"); // Upper-case file spec
defaultLangMap.add(".KTS", "kotlin"); // Upper-case file spec

command.add("--regex-kotlin=/^[[:space:]]*((abstract|final|sealed|implicit|lazy)[[:space:]]*)*" +
"(private[^ ]*|protected)?[[:space:]]*class[[:space:]]+([[:alnum:]_:]+)/\\4/c,classes/");
Expand Down Expand Up @@ -355,9 +349,6 @@ private void addClojureSupport(List<String> command) {
if (!env.getCtagsLanguages().contains("Clojure")) { // Built-in would be capitalized.
command.add("--langdef=clojure"); // Lower-case if user-defined.
}
defaultLangMap.add(".CLJ", "clojure"); // Upper-case file spec
defaultLangMap.add(".CLJS", "clojure"); // Upper-case file spec
defaultLangMap.add(".CLJX", "clojure"); // Upper-case file spec

command.add("--regex-clojure=/\\([[:space:]]*create-ns[[:space:]]+([-[:alnum:]*+!_:\\/.?]+)/\\1/n,namespace/");
command.add("--regex-clojure=/\\([[:space:]]*def[[:space:]]+([-[:alnum:]*+!_:\\/.?]+)/\\1/d,definition/");
Expand All @@ -375,8 +366,6 @@ private void addHaskellSupport(List<String> command) {
if (!env.getCtagsLanguages().contains("Haskell")) { // Built-in would be capitalized.
command.add("--langdef=haskell"); // below added with #912. Lowercase if user-defined.
}
defaultLangMap.add(".HS", "haskell"); // Upper-case file spec
defaultLangMap.add(".HSC", "haskell"); // Upper-case file spec

command.add("--regex-haskell=/^[[:space:]]*class[[:space:]]+([a-zA-Z0-9_]+)/\\1/c,classes/");
command.add("--regex-haskell=/^[[:space:]]*data[[:space:]]+([a-zA-Z0-9_]+)/\\1/t,types/");
Expand All @@ -392,7 +381,6 @@ private void addScalaSupport(List<String> command) {
if (!env.getCtagsLanguages().contains("Scala")) { // Built-in would be capitalized.
command.add("--langdef=scala"); // below is bug 61 to get full scala support. Lower-case
}
defaultLangMap.add(".SCALA", "scala"); // Upper-case file spec

command.add("--regex-scala=/^[[:space:]]*((abstract|final|sealed|implicit|lazy)[[:space:]]*)*" +
"(private|protected)?[[:space:]]*class[[:space:]]+([a-zA-Z0-9_]+)/\\4/c,classes/");
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* See LICENSE.txt included in this distribution for the specific
* language governing permissions and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at LICENSE.txt.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/

/*
* Copyright (c) 2017-2019, Chris Fraire <[email protected]>.
*/

package org.opengrok.indexer.analysis.asm;

import org.opengrok.indexer.analysis.AbstractAnalyzer;
import org.opengrok.indexer.analysis.AnalyzerFactory;
import org.opengrok.indexer.analysis.JFlexTokenizer;
import org.opengrok.indexer.analysis.JFlexXref;
import org.opengrok.indexer.analysis.plain.AbstractSourceCodeAnalyzer;

import java.io.Reader;

/**
* Represents an analyzer for assembly language.
*/
public class AsmAnalyzer extends AbstractSourceCodeAnalyzer {

/**
* Creates a new instance of {@link AsmAnalyzer}.
* @param factory instance
*/
protected AsmAnalyzer(AnalyzerFactory factory) {
super(factory, new JFlexTokenizer(new AsmSymbolTokenizer(AbstractAnalyzer.DUMMY_READER)));
}

/**
* @return {@code "Asm"}
*/
@Override
public String getCtagsLang() {
return "Asm";
}

/**
* Gets a version number to be used to tag processed documents so that
* re-analysis can be re-done later if a stored version number is different
* from the current implementation.
* @return 20191120_00
*/
@Override
protected int getSpecializedVersionNo() {
return 20191120_00; // Edit comment above too!
}

/**
* Creates a wrapped {@link AsmXref} instance.
* @return a defined instance
*/
@Override
protected JFlexXref newXref(Reader reader) {
return new JFlexXref(new AsmXref(reader));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* See LICENSE.txt included in this distribution for the specific
* language governing permissions and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at LICENSE.txt.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/

/*
* Copyright (c) 2017, 2019, Chris Fraire <[email protected]>.
*/

package org.opengrok.indexer.analysis.asm;

import org.opengrok.indexer.analysis.AbstractAnalyzer;
import org.opengrok.indexer.analysis.FileAnalyzerFactory;

/**
* Represents a factory to create {@link AsmAnalyzer} instances.
*/
public class AsmAnalyzerFactory extends FileAnalyzerFactory {

private static final String NAME = "Asm";

private static final String[] SUFFIXES = {"ASM", "S"};

/**
* Initializes a factory instance to associate file extensions ".asm" and
* ".s" with {@link AsmAnalyzer}.
*/
public AsmAnalyzerFactory() {
super(null, null, SUFFIXES, null, null, "text/plain", AbstractAnalyzer.Genre.PLAIN, NAME);
}

/**
* Creates a new {@link AsmAnalyzer} instance.
* @return a defined instance
*/
@Override
protected AbstractAnalyzer newAnalyzer() {
return new AsmAnalyzer(this);
}
}
Loading