4.x: Udpates to types and annotation processing (helidon-io#9168)

* APT codegen
- removing public methods from package private class
- deprecating all public types, as these should not be used outside of this module (only the processor is public, as it is a ServiceLoader provider implmentation)
- caching TypeInfo within a single annotation processing round
- now handles all processed types, to be able to trigger based on "meta" annotations

Class model:
Bugfix in class model, where content support used deprecated (and failing) method.
Added support for volatile fields

Types:
Updated with latest behavior of builder (mutated lists)

* Fix metadata codegen, to only store the file if there are modules available (i.e. never store empty array).

Author: tomas-langer

codegen/apt/src/main/java/io/helidon/codegen/apt/AptAnnotationFactory.java

@@ -17,15 +17,10 @@
 package io.helidon.codegen.apt;
 
 import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
 import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
 
 import javax.lang.model.element.AnnotationMirror;
 import javax.lang.model.element.AnnotationValue;
-import javax.lang.model.element.Element;
 import javax.lang.model.element.ExecutableElement;
 import javax.lang.model.util.Elements;
 
@@ -35,34 +30,11 @@
 /**
  * Factory for annotations.
  */
+@SuppressWarnings("removal")
 final class AptAnnotationFactory {
     private AptAnnotationFactory() {
     }
 
-    /**
-     * Creates a set of annotations using annotation processor.
-     *
-     * @param annoMirrors the annotation type mirrors
-     * @param elements annotation processing element utils
-     * @return the annotation value set
-     */
-    public static Set<Annotation> createAnnotations(List<? extends AnnotationMirror> annoMirrors, Elements elements) {
-        return annoMirrors.stream()
-                .map(it -> createAnnotation(it, elements))
-                .collect(Collectors.toCollection(LinkedHashSet::new));
-    }
-
-    /**
-     * Creates a set of annotations based using annotation processor.
-     *
-     * @param type the enclosing/owing type element
-     * @param elements annotation processing element utils
-     * @return the annotation value set
-     */
-    public static Set<Annotation> createAnnotations(Element type, Elements elements) {
-        return createAnnotations(type.getAnnotationMirrors(), elements);
-    }
-
     /**
      * Creates an instance from an annotation mirror during annotation processing.
      *

codegen/apt/src/main/java/io/helidon/codegen/apt/AptContext.java

@@ -16,16 +16,22 @@
 
 package io.helidon.codegen.apt;
 
+import java.util.Optional;
 import java.util.Set;
+import java.util.function.Supplier;
 
 import javax.annotation.processing.ProcessingEnvironment;
 
 import io.helidon.codegen.CodegenContext;
 import io.helidon.codegen.Option;
+import io.helidon.common.types.TypeInfo;
+import io.helidon.common.types.TypeName;
 
 /**
  * Annotation processing code generation context.
+ * @deprecated this API will be package local in the future, use through Helidon codegen only
  */
+@Deprecated(forRemoval = true, since = "4.1.0")
 public interface AptContext extends CodegenContext {
     /**
      * Create context from the processing environment, and a set of additional supported options.
@@ -44,4 +50,14 @@ static AptContext create(ProcessingEnvironment env, Set<Option<?>> options) {
      * @return environment
      */
     ProcessingEnvironment aptEnv();
+
+    /**
+     * Get a cached instance of the type info, and if not cached, cache the provided one.
+     * Only type infos known not to be modified during this build are cached.
+     *
+     * @param typeName type name
+     * @param typeInfoSupplier supplier of value if it is not yet cached
+     * @return type info for that name, in case the type info cannot be created, an empty optional
+     */
+    Optional<TypeInfo> cache(TypeName typeName, Supplier<Optional<TypeInfo>> typeInfoSupplier);
 }

codegen/apt/src/main/java/io/helidon/codegen/apt/AptContextImpl.java

@@ -19,9 +19,12 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Predicate;
+import java.util.function.Supplier;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -40,11 +43,14 @@
 import io.helidon.common.types.TypeName;
 import io.helidon.common.types.TypedElementInfo;
 
+@SuppressWarnings("removal")
 class AptContextImpl extends CodegenContextBase implements AptContext {
     private static final Pattern SCOPE_PATTERN = Pattern.compile("(\\w+).*classes");
 
     private final ProcessingEnvironment env;
     private final ModuleInfo moduleInfo;
+    private final Map<TypeName, Optional<TypeInfo>> safeTypeCache = new HashMap<>();
+    private final Map<TypeName, Optional<TypeInfo>> typeCache = new HashMap<>();
 
     AptContextImpl(ProcessingEnvironment env,
                    CodegenOptions options,
@@ -59,7 +65,7 @@ class AptContextImpl extends CodegenContextBase implements AptContext {
         this.moduleInfo = moduleInfo;
     }
 
-    static AptContext create(ProcessingEnvironment env, Set<Option<?>> supportedOptions) {
+    static AptContextImpl create(ProcessingEnvironment env, Set<Option<?>> supportedOptions) {
         CodegenOptions options = AptOptions.create(env);
 
         CodegenScope scope = guessScope(env, options);
@@ -81,11 +87,16 @@ public ProcessingEnvironment aptEnv() {
 
     @Override
     public Optional<TypeInfo> typeInfo(TypeName typeName) {
+        if (typeCache.containsKey(typeName)) {
+            return typeCache.get(typeName);
+        }
+        // cached by the factory
         return AptTypeInfoFactory.create(this, typeName);
     }
 
     @Override
     public Optional<TypeInfo> typeInfo(TypeName typeName, Predicate<TypedElementInfo> elementPredicate) {
+        // cannot be cached
         return AptTypeInfoFactory.create(this, typeName, elementPredicate);
     }
 
@@ -94,6 +105,39 @@ public Optional<ModuleInfo> module() {
         return Optional.ofNullable(moduleInfo);
     }
 
+    @Override
+    public Optional<TypeInfo> cache(TypeName typeName, Supplier<Optional<TypeInfo>> typeInfoSupplier) {
+        if (typeName.generic() || !typeName.typeArguments().isEmpty() || !typeName.typeParameters().isEmpty()) {
+            // generic types cannot be cached
+            return typeInfoSupplier.get();
+        }
+
+        if (typeName.packageName().startsWith("java.")
+                || typeName.packageName().startsWith("javax.")
+                || typeName.packageName().startsWith("sun.")
+                || typeName.packageName().startsWith("com.sun")) {
+            Optional<TypeInfo> typeInfo = safeTypeCache.get(typeName);
+            if (typeInfo != null) {
+                return typeInfo;
+            }
+            typeInfo = typeInfoSupplier.get();
+            safeTypeCache.put(typeName, typeInfo);
+            return typeInfo;
+        }
+
+        Optional<TypeInfo> typeInfo = typeCache.get(typeName);
+        if (typeInfo != null) {
+            return typeInfo;
+        }
+        typeInfo = typeInfoSupplier.get();
+        typeCache.put(typeName, typeInfo);
+        return typeInfo;
+    }
+
+    void resetCache() {
+        typeCache.clear();
+    }
+
     private static Optional<ModuleInfo> findModule(Filer filer) {
         // expected is source location
         try {

codegen/apt/src/main/java/io/helidon/codegen/apt/AptProcessor.java

@@ -17,12 +17,12 @@
 package io.helidon.codegen.apt;
 
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import javax.annotation.processing.AbstractProcessor;
 import javax.annotation.processing.ProcessingEnvironment;
@@ -35,7 +35,9 @@
 
 import io.helidon.codegen.Codegen;
 import io.helidon.codegen.CodegenEvent;
+import io.helidon.codegen.CodegenException;
 import io.helidon.codegen.Option;
+import io.helidon.common.types.Annotation;
 import io.helidon.common.types.TypeInfo;
 import io.helidon.common.types.TypeName;
 
@@ -45,10 +47,11 @@
 /**
  * Annotation processor that maps APT types to Helidon types, and invokes {@link io.helidon.codegen.Codegen}.
  */
+@SuppressWarnings("removal")
 public final class AptProcessor extends AbstractProcessor {
     private static final TypeName GENERATOR = TypeName.create(AptProcessor.class);
 
-    private AptContext ctx;
+    private AptContextImpl ctx;
     private Codegen codegen;
 
     /**
@@ -66,13 +69,8 @@ public SourceVersion getSupportedSourceVersion() {
 
     @Override
     public Set<String> getSupportedAnnotationTypes() {
-        return Stream.concat(codegen.supportedAnnotations()
-                                     .stream()
-                                     .map(TypeName::fqName),
-                             codegen.supportedAnnotationPackagePrefixes()
-                                     .stream()
-                                     .map(it -> it + "*"))
-                .collect(Collectors.toSet());
+        // we need to support all annotations, to be able to use meta-annotations
+        return Set.of("*");
     }
 
     @Override
@@ -87,12 +85,14 @@ public Set<String> getSupportedOptions() {
     public synchronized void init(ProcessingEnvironment processingEnv) {
         super.init(processingEnv);
 
-        this.ctx = AptContext.create(processingEnv, Codegen.supportedOptions());
+        this.ctx = AptContextImpl.create(processingEnv, Codegen.supportedOptions());
         this.codegen = Codegen.create(ctx, GENERATOR);
     }
 
     @Override
     public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment roundEnv) {
+        this.ctx.resetCache();
+
         Thread thread = Thread.currentThread();
         ClassLoader previousClassloader = thread.getContextClassLoader();
         thread.setContextClassLoader(AptProcessor.class.getClassLoader());
@@ -102,6 +102,19 @@ public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment
         try {
             doProcess(annotations, roundEnv);
             return true;
+        } catch (CodegenException e) {
+            Object originatingElement = e.originatingElement()
+                    .orElse(null);
+            if (originatingElement instanceof Element element) {
+                processingEnv.getMessager().printError(e.getMessage(), element);
+            } else if (originatingElement instanceof TypeName typeName) {
+                processingEnv.getMessager().printError(e.getMessage() + ", source: " + typeName.fqName());
+            } else {
+                if (originatingElement != null) {
+                    processingEnv.getMessager().printError(e.getMessage() + ", source: " + originatingElement);
+                }
+            }
+            throw e;
         } finally {
             thread.setContextClassLoader(previousClassloader);
         }
@@ -115,32 +128,96 @@ private void doProcess(Set<? extends TypeElement> annotations, RoundEnvironment
             return;
         }
 
-        if (annotations.isEmpty()) {
+        Set<UsedAnnotation> usedAnnotations = usedAnnotations(annotations);
+
+        if (usedAnnotations.isEmpty()) {
             // no annotations, no types, still call the codegen, maybe it has something to do
             codegen.process(List.of());
             return;
         }
 
-        List<TypeInfo> allTypes = discoverTypes(annotations, roundEnv);
+        List<TypeInfo> allTypes = discoverTypes(usedAnnotations, roundEnv);
         codegen.process(allTypes);
     }
 
-    private List<TypeInfo> discoverTypes(Set<? extends TypeElement> annotations, RoundEnvironment roundEnv) {
+    private Set<UsedAnnotation> usedAnnotations(Set<? extends TypeElement> annotations) {
+        var exactTypes = codegen.supportedAnnotations()
+                .stream()
+                .map(TypeName::fqName)
+                .collect(Collectors.toSet());
+        var prefixes = codegen.supportedAnnotationPackagePrefixes();
+
+        Set<UsedAnnotation> result = new HashSet<>();
+
+        for (TypeElement annotation : annotations) {
+            TypeName typeName = TypeName.create(annotation.getQualifiedName().toString());
+
+            /*
+            find meta annotations that are supported:
+            - annotation that annotates the current annotation
+            */
+            Set<TypeName> supportedAnnotations = new HashSet<>();
+            if (supportedAnnotation(exactTypes, prefixes, typeName)) {
+                supportedAnnotations.add(typeName);
+            }
+            addSupportedAnnotations(exactTypes, prefixes, supportedAnnotations, typeName);
+            if (!supportedAnnotations.isEmpty()) {
+                result.add(new UsedAnnotation(typeName, annotation, supportedAnnotations));
+            }
+        }
+
+        return result;
+    }
+
+    private boolean supportedAnnotation(Set<String> exactTypes, Set<String> prefixes, TypeName annotationType) {
+        if (exactTypes.contains(annotationType.fqName())) {
+            return true;
+        }
+        String packagePrefix = annotationType.packageName() + ".";
+        for (String prefix : prefixes) {
+            if (packagePrefix.startsWith(prefix)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private void addSupportedAnnotations(Set<String> exactTypes,
+                                         Set<String> prefixes,
+                                         Set<TypeName> supportedAnnotations,
+                                         TypeName annotationType) {
+        Optional<TypeInfo> foundInfo = AptTypeInfoFactory.create(ctx, annotationType);
+        if (foundInfo.isPresent()) {
+            TypeInfo annotationInfo = foundInfo.get();
+            List<Annotation> annotations = annotationInfo.annotations();
+            for (Annotation annotation : annotations) {
+                TypeName typeName = annotation.typeName();
+                if (supportedAnnotation(exactTypes, prefixes, typeName)) {
+                    if (supportedAnnotations.add(typeName)) {
+                        addSupportedAnnotations(exactTypes, prefixes, supportedAnnotations, typeName);
+                    }
+                }
+            }
+        }
+    }
+
+    private List<TypeInfo> discoverTypes(Set<UsedAnnotation> annotations, RoundEnvironment roundEnv) {
         // we must discover all types that should be handled, create TypeInfo and only then check if these should be processed
         // as we may replace annotations, elements, and whole types.
 
         // first collect all types (group by type name, so we do not have duplicity)
         Map<TypeName, TypeElement> types = new HashMap<>();
 
-        for (TypeElement annotation : annotations) {
-            Set<? extends Element> elementsAnnotatedWith = roundEnv.getElementsAnnotatedWith(annotation);
+        for (UsedAnnotation annotation : annotations) {
+            TypeElement annotationElement = annotation.annotationElement();
+            Set<? extends Element> elementsAnnotatedWith = roundEnv.getElementsAnnotatedWith(annotationElement);
             for (Element element : elementsAnnotatedWith) {
                 ElementKind kind = element.getKind();
                 switch (kind) {
-                case ENUM, INTERFACE, CLASS, ANNOTATION_TYPE, RECORD -> addType(types, element, element, annotation);
+                case ENUM, INTERFACE, CLASS, ANNOTATION_TYPE, RECORD -> addType(types, element, element, annotationElement);
                 case ENUM_CONSTANT, CONSTRUCTOR, METHOD, FIELD, STATIC_INIT, INSTANCE_INIT, RECORD_COMPONENT ->
-                        addType(types, element.getEnclosingElement(), element, annotation);
-                case PARAMETER -> addType(types, element.getEnclosingElement().getEnclosingElement(), element, annotation);
+                        addType(types, element.getEnclosingElement(), element, annotationElement);
+                case PARAMETER -> addType(types, element.getEnclosingElement().getEnclosingElement(), element, annotationElement);
                 default -> ctx.logger().log(TRACE, "Ignoring annotated element, not supported: " + element + ", kind: " + kind);
                 }
             }
@@ -177,4 +254,16 @@ private void addType(Map<TypeName, TypeElement> types,
                                                      processedElement);
         }
     }
+
+    /**
+     * Annotation that annotates a processed type and that must be processed.
+     *
+     * @param annotationType       annotation on processed type
+     * @param annotationElement    element of the annotation
+     * @param supportedAnnotations annotations that are supported (either the actual annotation, or meta-annotations)
+     */
+    private record UsedAnnotation(TypeName annotationType,
+                                  TypeElement annotationElement,
+                                  Set<TypeName> supportedAnnotations) {
+    }
 }