Using the new Doclet API

Introduction

In JDK 9, JEP 221 introduced a new "Doclet API" to supersede the previous API, which was becoming increasing difficult to update to support new language features. The new API leverages other Java SE and JDK API, such as the Language Model API, added in JDK 6, and the Compiler Tree API, added in JDK 6 and extended in JDK 8.

While there is some correspondence between features in the old API and features in the new API and the APIs that it utilizes, the new API is generally more powerful and uses some different idioms, so that the correspondence is not directly one-to-one.

This note describes various aspects of the overall new API, illustrating the usage with a series of simple examples.

A simple Doclet

The following code shows a minimal doclet that just prints out the names of elements specified on the command line.

A minimal doclet
package tips;
 
import java.util.Collections;
import java.util.Locale;
import java.util.Set;
import javax.lang.model.SourceVersion;
 
import jdk.javadoc.doclet.Doclet;
import jdk.javadoc.doclet.DocletEnvironment;
import jdk.javadoc.doclet.Reporter;
 
/**
 * A minimal doclet that just prints out the names of the
 * selected elements.
 */
public class BasicDoclet implements Doclet {
    @Override
    public void init(Locale locale, Reporter reporter) {  }
 
    @Override
    public String getName() {
        // For this doclet, the name of the doclet is just the
        // simple name of the class. The name may be used in
        // messages related to this doclet, such as in command-line
        // help when doclet-specific options are provided.
        return getClass().getSimpleName();
    }
 
    @Override
    public Set<? extends Option> getSupportedOptions() {
        // This doclet does not support any options.
        return Collections.emptySet();
    }
 
    @Override
    public SourceVersion getSupportedSourceVersion() {
        // This doclet supports all source versions.
        // More sophisticated doclets may use a more
        // specific version, to ensure that they do not
        // encounter more recent language features that
        // they may not be able to handle.
        return SourceVersion.latest();
    }
 
    private static final boolean OK = true;
 
    @Override
    public boolean run(DocletEnvironment environment) {
        // This method is called to perform the work of the doclet.
        // In this case, it just prints out the names of the
        // elements specified on the command line.
        environment.getSpecifiedElements()
                .forEach(System.out::println);
        return OK;
    }
}

Here is an example of how to run the doclet and the output it generates.

Running BasicDoclet
$ /opt/jdk/11/bin/javadoc \
    -docletpath classes \
    -doclet tips.BasicDoclet \
    src/tips/BasicDoclet.java 
Constructing Javadoc information...
tips.BasicDoclet

Options

A common requirement of many doclets is to be able to handle doclet-specific options. By implementing the method Doclet.getSupportedOptions you can declare the options supported by a doclet. Options are represented by instances of Doclet.Option. This is an interface, and so it is common to declare an abstract class that implements the interface, and to use anonymous subclasses to implement the run method, to handle occurrences of the option found on the command line.

The following example shows how to declare an option that takes no arguments, an option that takes a string-valued argument, and an option that takes an numeric argument, generating an error if an invalid value is found.

Declaring options for a doclet
package tips;
 
import java.util.List;
import java.util.Locale;
import java.util.Set;
import javax.lang.model.SourceVersion;
 
import jdk.javadoc.doclet.Doclet;
import jdk.javadoc.doclet.DocletEnvironment;
import jdk.javadoc.doclet.Reporter;
 
/**
 * A doclet to illustrate the use of doclet-specific options.
 *
 * The doclet simply prints of the values of the declared options,
 * whether or not given explicitly on the command line.
 */
public class OptionsDoclet implements Doclet {
    private static final boolean OK = true;
 
    private boolean alpha;
    private String beta;
    private int gamma;
 
    /**
     * A base class for declaring options.
     * Subtypes for specific options should implement
     * the {@link #process(String,List) process} method
     * to handle instances of the option found on the
     * command line.
     */
    abstract class Option implements Doclet.Option {
        private final String name;
        private final boolean hasArg;
        private final String description;
        private final String parameters;
 
        Option(String name, boolean hasArg,
               String description, String parameters) {
            this.name = name;
            this.hasArg = hasArg;
            this.description = description;
            this.parameters = parameters;
        }
 
        @Override
        public int getArgumentCount() {
            return hasArg ? 1 : 0;
        }
 
        @Override
        public String getDescription() {
            return description;
        }
 
        @Override
        public Kind getKind() {
            return Kind.STANDARD;
        }
 
        @Override
        public List<String> getNames() {
            return List.of(name);
        }
 
        @Override
        public String getParameters() {
            return hasArg ? parameters : null;
        }
    }
 
    private final Set<Option> options = Set.of(
            // An option that takes no arguments.
            new Option("--alpha", false, "a flag", null) {
                @Override
                public boolean process(String option,
                                       List<String> arguments) {
                    alpha = true;
                    return OK;
                }
            },
 
            // An option that takes a single string-valued argument.
            new Option("--beta", true, "an option", "<string>") {
                @Override
                public boolean process(String option,
                                       List<String> arguments) {
                    beta = arguments.get(0);
                    return OK;
                }
            },
 
            // An option that takes a single integer-valued srgument.
            new Option("--gamma", true, "another option", "<int>") {
                @Override
                public boolean process(String option,
                                       List<String> arguments) {
                    String arg = arguments.get(0);
                    try {
                        gamma = Integer.parseInt(arg);
                        return OK;
                    } catch (NumberFormatException e) {
                        // Note: it would be better to use
                        // {@link Reporter} to print an error message,
                        // so that the javadoc tool "knows" that an
                        // error was reported in conjunction\ with
                        // the "return false;" that follows.
                        System.err.println("not an int: " + arg);
                        return false;
                    }
                }
            }
    );
 
 
    @Override
    public void init(Locale locale, Reporter reporter) {  }
 
    @Override
    public String getName() {
        return getClass().getSimpleName();
    }
 
    @Override
    public Set<? extends Option> getSupportedOptions() {
        return options;
    }
 
    @Override
    public SourceVersion getSupportedSourceVersion() {
        return SourceVersion.latest();
    }
 
 
    @Override
    public boolean run(DocletEnvironment environment) {
        System.out.println("alpha: " + alpha);
        System.out.println("beta: " + beta);
        System.out.println("gamma: " + gamma);
        return OK;
    }
}

When the doclet is executed with valid options, it will print out the specified or default values of alpha, beta and gamma. If an invalid value is provided for gamma, an error message will be generated. If you use the javadoc --help, it will display the command-line help, including the standard options supported by the tool, and any custom options supported by the doclet.

Reporters

The preceding examples just write directly to System.out. Using a Reporter, you can generate messages that are associated with an element or a position in a documentation comment. The javadoc tool will try to identify the appropriate source file and line within the source file and will include that in the message displayed to the user.

The following example uses the reporter to report the kind of the elements specified on the command line.

Using a reporter
package tips;
 
import java.util.Collections;
import java.util.Locale;
import java.util.Set;
import javax.lang.model.SourceVersion;
import javax.tools.Diagnostic;
 
import jdk.javadoc.doclet.Doclet;
import jdk.javadoc.doclet.DocletEnvironment;
import jdk.javadoc.doclet.Reporter;
 
/**
 * A doclet to illustate the use of a {@link Reporter}.
 *
 * The doclet uses a reporter to print the name of the
 * selected types on the command line.
 *
 * Note: some versions of javadoc may incorrectly generate
 * warnings instead of notes. Bug JDK--8224083.
 */
public class ReporterDoclet implements Doclet {
    private Reporter reporter;
 
    @Override
    public void init(Locale locale, Reporter reporter) {
        this.reporter = reporter;
    }
 
    @Override
    public String getName() {
        return getClass().getSimpleName();
    }
 
    @Override
    public Set<? extends Option> getSupportedOptions() {
        return Collections.emptySet();
    }
 
    @Override
    public SourceVersion getSupportedSourceVersion() {
        return SourceVersion.latest();
    }
 
    private static final boolean OK = true;
 
    @Override
    public boolean run(DocletEnvironment environment) {
        environment.getSpecifiedElements().forEach(e ->
                reporter.print(Diagnostic.Kind.NOTE,
                        e,
                        e.getKind().toString()));
        return OK;
    }
}

When run with the source code as an input file, it generates output like the following:

    src/tips/ReporterDoclet.java:13: CLASS

Notice the filename and line number, identifying the element, preceding the text supplied by the doclet.

There's a bug in the current implementation, and notes actually get generated as warnings.

Visitors and Scanners

The Language Model API and Compiler Tree API each define a number of class hierarchies, with subtypes of a common supertype representing different kinds of the supertype.

To implement operations based on the class of any of item in these hierarchies, you can use a corresponding visitor class for the hierarchy. Visitors are a programming pattern that allow you to provide different behaviors for different subtypes of the base type.

Class Hierarchies
API Base Type Description Visitor
Language Model API Element Program elements ElementVisitor
Language Model API TypeMirror Types of and within program elements TypeVisitor
Language Model API AnnotationValue Values within an annotation AnnotationValueVisitor
Compiler Tree API Tree Nodes within the abstract syntax tree (AST) for a source file (Typically not used within doclets.) TreeVisitor
Compiler Tree API DocTree Nodes within the abstract syntax tree (AST) for a documentation comment in a source file. DocTreeVisitor

For each of the basic visitor classes, various subtypes are provided for convenience. For more details, see the appropriate API documentation.

The basic visitor mechanism just processes a single instance of any of the base types. For most of the visitor classes, there is a special subtype, called a scanner. By default, scanner classes recursively visit a node and all of its children, although the behavior can be modified by overriding the methods for any kind of node, as desired. Scanner classes are particularly useful for traversing a hierarchy of nodes, looking for nodes with given properties, such as all methods within a class and its nested classes.

OpenJDK Project Amber is exploring the use of a new language feature called patterns that will help avoid much of the boilerplate found in typical use of the visitor classes. See also Pattern Matching for Java.

The following example uses a combination of an ElementScanner and a DocTreeScanner to display the elements and documentation comments found in the elements specified in the command line.

Using scanners to show elements and documention comments
package tips;
 
import java.io.PrintStream;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import javax.lang.model.SourceVersion;
import javax.lang.model.element.Element;
import javax.lang.model.util.ElementScanner9;
import javax.tools.Diagnostic;
 
import com.sun.source.doctree.DocCommentTree;
import com.sun.source.doctree.DocTree;
import com.sun.source.util.DocTreeScanner;
import com.sun.source.util.DocTrees;
import jdk.javadoc.doclet.Doclet;
import jdk.javadoc.doclet.DocletEnvironment;
import jdk.javadoc.doclet.Reporter;
 
/**
 * A doclet to demonstrate the use of {@link ElementScanner9}
 * and {@link DocTreeScanner}.
 *
 * @version 1.0
 * @author Duke
 */
public class ScannerDoclet implements Doclet {
    private static final boolean OK = true;
    private static final boolean FAILED= false;
 
    private boolean showElements;
    private boolean showComments;
 
    private Reporter reporter;
    private DocTrees treeUtils;
 
    abstract class Option implements Doclet.Option {
        private final String name;
        private final boolean hasArg;
        private final String description;
        private final String parameters;
 
        Option(String name, boolean hasArg,
               String description, String parameters) {
            this.name = name;
            this.hasArg = hasArg;
            this.description = description;
            this.parameters = parameters;
        }
 
        @Override
        public int getArgumentCount() {
            return hasArg ? 1 : 0;
        }
 
        @Override
        public String getDescription() {
            return description;
        }
 
        @Override
        public Kind getKind() {
            return Kind.STANDARD;
        }
 
        @Override
        public List<String> getNames() {
            return List.of(name);
        }
 
        @Override
        public String getParameters() {
            return hasArg ? parameters : null;
        }
    }
 
    private final Set<Option> options = Set.of(
            new Option("--show-elements", false,
                    "show selected elements", null) {
                @Override
                public boolean process(String option,
                                       List<String> arguments) {
                    showElements = true;
                    return OK;
                }
            },
            new Option("--show-comments", false,
                    "show comments", null) {
                @Override
                public boolean process(String option,
                                       List<String> arguments) {
                    showComments = true;
                    return OK;
                }
            }
    );
 
 
    @Override
    public void init(Locale locale, Reporter reporter) {
        this.reporter = reporter;
    }
 
    @Override
    public String getName() {
        return getClass().getSimpleName();
    }
 
    @Override
    public Set<? extends Option> getSupportedOptions() {
        return options;
    }
 
    @Override
    public SourceVersion getSupportedSourceVersion() {
        return SourceVersion.latest();
    }
 
 
    @Override
    public boolean run(DocletEnvironment environment) {
        if (!showElements && !showComments) {
            reporter.print(Diagnostic.Kind.ERROR,
                    "specify either --show-elements or --show-comments");
            return FAILED;
        }
        treeUtils = environment.getDocTrees();
        ShowElements se = new ShowElements(System.out);
        se.show(environment.getIncludedElements());
        return OK;
    }
 
    /**
     * A scanner to display the structure of a series of elements
     * and their documentation comments.
     */
    class ShowElements extends ElementScanner9<Void, Integer> {
        final PrintStream out;
 
        ShowElements(PrintStream out) {
            this.out = out;
        }
 
        void show(Set<? extends Element> elements) {
            scan(elements, 0);
        }
 
        @Override
        public Void scan(Element e, Integer depth) {
            DocCommentTree dcTree = treeUtils.getDocCommentTree(e);
            String indent = "  ".repeat(depth);
            if (showElements || showComments && dcTree != null) {
                out.println(indent + "| " + e.getKind() + " " + e);
            }
            if (showComments && dcTree != null) {
                new ShowDocTrees(out).scan(dcTree, depth + 1);
            }
            return super.scan(e, depth + 1);
        }
    }
 
    /**
     * A scanner to display the structure of a documentation comment.
     */
    class ShowDocTrees extends DocTreeScanner<Void, Integer> {
        final PrintStream out;
 
        ShowDocTrees(PrintStream out) {
            this.out = out;
        }
 
        @Override
        public Void scan(DocTree t, Integer depth) {
            String indent = "  ".repeat(depth);
            out.println(indent + "# "
                    + t.getKind() + " "
                    + t.toString().replace("\n", "\n" + indent + "#    "));
            return super.scan(t, depth + 1);
        }
    }
}

Elements and Type Mirrors

Elements generally represent the declaration of an item, such as a module, package, type or member, whereas type mirrors generally represent the use of (or reference to) an element, that may appear within another type mirror or an element. The difference is most notable for declarations of generic types: for example, interface List<T> declares an element which represents a family of type mirrors, of which List<String> is just a single instance. The difference can also be seen in the context of annotations: annotations on a type mirror represent the annotations found on the specific use of the type, whereas annotations on the corresponding element represent the annotations found on the declaration of the type.

There is obviously a certain duality between elements and type mirrors, and it is possible to move between the two class hierarchies with methods like Element.asType() and Types.asElement(TypeMirror).

The following example shows how to visit the members of a type, and to examine the type mirrors that are part of the declarations of those members, and to find the elements to which those type mirrors correspond.

Using elements and type mirrors
package tips;
 
import java.io.PrintStream;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import javax.lang.model.SourceVersion;
import javax.lang.model.element.Element;
import javax.lang.model.element.ElementKind;
import javax.lang.model.element.ExecutableElement;
import javax.lang.model.element.VariableElement;
import javax.lang.model.type.TypeMirror;
import javax.lang.model.util.ElementScanner9;
import javax.lang.model.util.Types;
 
import jdk.javadoc.doclet.Doclet;
import jdk.javadoc.doclet.DocletEnvironment;
import jdk.javadoc.doclet.Reporter;
 
/**
 * A doclet to demonstrate the use of {@link TypeMirror
 * type mirrors}, and the relationship to the corresponding
 * {@link Element elements}.
 */
public class TypeMirrorDoclet implements Doclet {
    private static final boolean OK = true;
    private Types typeUtils;
 
    @Override
    public void init(Locale locale, Reporter reporter) { }
 
    @Override
    public String getName() {
        return getClass().getSimpleName();
    }
 
    @Override
    public Set<? extends Option> getSupportedOptions() {
        return Collections.emptySet();
    }
 
    @Override
    public SourceVersion getSupportedSourceVersion() {
        return SourceVersion.latest();
    }
 
 
    @Override
    public boolean run(DocletEnvironment environment) {
        typeUtils = environment.getTypeUtils();
 
        ShowElements se = new ShowElements(System.out);
        se.show(environment.getSpecifiedElements());
        return OK;
    }
 
    class ShowElements extends ElementScanner9<Void, Integer> {
        final PrintStream out;
 
        ShowElements(PrintStream out) {
            this.out = out;
        }
 
        void show(Set<? extends Element> elements) {
            scan(elements, 0);
        }
        @Override
        public Void scan(Element e, Integer depth) {
            return super.scan(e, depth + 1);
        }
 
        @Override
        public Void visitExecutable(ExecutableElement ee, Integer depth) {
            String indent = "  ".repeat(depth);
            out.println(indent + ee.getKind() + ": " + ee);
            if (!ee.getTypeParameters().isEmpty()) {
                out.println(indent + "[Type Parameters]");
                scan(ee.getTypeParameters(), depth);
            }
            if (ee.getKind() == ElementKind.METHOD) {
                show("Return type", ee.getReturnType(), depth);
            }
            if (!ee.getParameters().isEmpty()) {
                out.println(indent + "[Parameters]");
                scan(ee.getParameters(), depth);
            }
            show("Throws", ee.getThrownTypes(), depth);
 
            return super.visitExecutable(ee, depth);
        }
 
        @Override
        public Void visitVariable(VariableElement ve, Integer depth) {
            if (ve.getKind() == ElementKind.PARAMETER) {
                String indent = "  ".repeat(depth);
                out.println(indent + ve.getKind() + ": " + ve);
                show("Type", ve.asType(), depth);
            }
            return super.visitVariable(ve, depth);
        }
 
        private void show(String label, List<? extends TypeMirror> list,
                          int depth) {
            if (!list.isEmpty()) {
                String indent = "  ".repeat(depth);
                out.println(indent + "[" + label + "]");
                int i = 0;
                for (TypeMirror tm : list) {
                    show("#" + (i++), tm, depth + 1);
                }
            }
        }
 
        private void show(String label, TypeMirror tm, int depth) {
            String indent = "  ".repeat(depth);
            out.println(indent + "[" + label + "]");
            out.println(indent + "  TypeMirror: " + tm);
            out.println(indent + "  as Element: " + typeUtils.asElement(tm));
        }
    }
}

Accessing Tags

Compared to the old Doclet API, the new Doclet API provides a richer structure to represent documentation comments, rooted in DocCommentTree. However, for some doclets that just wish to process selected block tags within the comment, a simpler model may be more desirable.

The following example shows how to use a visitor to access individual custom tags within a documentation comment, returning the tags in a Map<String, List<String>>. The example could obviously be extended to cover selected standard tags, and to provide other information about each tag instance, such as the position within the enclosing source file.

Accessing the custom tags in a comment
package tips;
 
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import javax.lang.model.SourceVersion;
import javax.lang.model.element.Element;
import javax.lang.model.util.ElementScanner9;
 
import com.sun.source.doctree.DocCommentTree;
import com.sun.source.doctree.UnknownBlockTagTree;
import com.sun.source.util.DocTrees;
import com.sun.source.util.SimpleDocTreeVisitor;
import jdk.javadoc.doclet.Doclet;
import jdk.javadoc.doclet.DocletEnvironment;
import jdk.javadoc.doclet.Reporter;
 
/**
 * A doclet that shows how to access custom tags in a
 * documentatio comment.
 *
 * @tag1 a b c
 * @tag2 name=value
 * @tag1 d e f
 */
public class TagScannerDoclet implements Doclet {
    private static final boolean OK = true;
 
    private DocTrees treeUtils;
 
    @Override
    public void init(Locale locale, Reporter reporter) { }
 
    @Override
    public String getName() {
        return getClass().getSimpleName();
    }
 
    @Override
    public Set<? extends Option> getSupportedOptions() {
        return Collections.emptySet();
    }
 
    @Override
    public SourceVersion getSupportedSourceVersion() {
        return SourceVersion.latest();
    }
 
 
    @Override
    public boolean run(DocletEnvironment environment) {
        treeUtils = environment.getDocTrees();
        ShowTags st = new ShowTags(System.out);
        st.show(environment.getSpecifiedElements());
        return OK;
    }
 
    /**
     * A scanner to search for elements with documentation comments,
     * and to examine those comments for custom tags.
     */
    class ShowTags extends ElementScanner9<Void, Integer> {
        final PrintStream out;
 
        ShowTags(PrintStream out) {
            this.out = out;
        }
 
        void show(Set<? extends Element> elements) {
            scan(elements, 0);
        }
 
        @Override
        public Void scan(Element e, Integer depth) {
            DocCommentTree dcTree = treeUtils.getDocCommentTree(e);
            if (dcTree != null) {
                String indent = "  ".repeat(depth);
                out.println(indent + "| " + e.getKind() + " " + e);
                Map<String, List<String>> tags = new TreeMap<>();
                new TagScanner(tags).visit(dcTree, null);
                tags.forEach((t,l) -> {
                    out.println(indent + "  @" + t);
                    l.forEach(c -> out.println(indent + "    " + c));
                });
            }
            return super.scan(e, depth + 1);
        }
    }
 
    /**
     * A visitor to gather the block tags found in a comment.
     */
    class TagScanner extends SimpleDocTreeVisitor<Void, Void> {
        private final Map<String, List<String>> tags;
 
        TagScanner(Map<String, List<String>> tags) {
            this.tags = tags;
        }
 
        @Override
        public Void visitDocComment(DocCommentTree tree, Void p) {
            return visit(tree.getBlockTags(), null);
        }
 
        @Override
        public Void visitUnknownBlockTag(UnknownBlockTagTree tree,
                                         Void p) {
            String name = tree.getTagName();
            String content = tree.getContent().toString();
            tags.computeIfAbsent(name,
                    n -> new ArrayList<>()).add(content);
            return null;
        }
    }
}

Accessing File Objects

It is not currently possible to directly access the source or class file containing the declaration of an element. (See JDK-8224922.) However, for those cases where the declaration is in a source file, you can access the information indirectly, via the TreePath object in the DocTree API, as shown in the following example.

Accessing the file for an element
package tips;
 
import java.io.PrintStream;
import java.util.Collections;
import java.util.Locale;
import java.util.Set;
import javax.lang.model.SourceVersion;
import javax.lang.model.element.Element;
import javax.lang.model.element.TypeElement;
import javax.lang.model.util.ElementScanner9;
import javax.tools.JavaFileObject;
 
import com.sun.source.util.DocTrees;
import com.sun.source.util.TreePath;
import jdk.javadoc.doclet.Doclet;
import jdk.javadoc.doclet.DocletEnvironment;
import jdk.javadoc.doclet.Reporter;
 
/**
 * A doclet that prints out the file objects for the selected types.
 */
public class FileObjectDoclet implements Doclet {
    DocTrees treeUtils;
 
    @Override
    public void init(Locale locale, Reporter reporter) {  }
 
    @Override
    public String getName() {
        return getClass().getSimpleName();
    }
 
    @Override
    public Set<? extends Option> getSupportedOptions() {
        return Collections.emptySet();
    }
 
    @Override
    public SourceVersion getSupportedSourceVersion() {
        return SourceVersion.latest();
    }
 
    private static final boolean OK = true;
 
    @Override
    public boolean run(DocletEnvironment environment) {
        treeUtils = environment.getDocTrees();
        new ShowFileObjects(System.out)
                .show(environment.getSpecifiedElements());
        return OK;
    }
 
    /**
     * A scanner that displays the name of the source file
     * (if available) for any types that it encounters.
     */
    class ShowFileObjects extends ElementScanner9<Void, Void> {
        final PrintStream out;
 
        ShowFileObjects(PrintStream out) {
            this.out = out;
        }
 
        void show(Set<? extends Element> elements) {
            scan(elements, null);
        }
 
        @Override
        public Void visitType(TypeElement te, Void p) {
            TreePath dct = treeUtils.getPath(te);
            if (dct == null) {
                out.println(te + ": no source file found");
            } else {
                JavaFileObject fo =
                        dct.getCompilationUnit().getSourceFile();
                out.println(te + ": " + fo.getName());
            }
            return super.visitType(te, p);
        }
    }
}