diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index e69de29bb..debb54d4e 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -0,0 +1,13 @@ +* Loader: Add missing packages to CQL list +* Loader: fix up language/local loading sequence +* Loader: fix bug for cross-version extension containing escaped [x] +* Validator: Add check for duplicate ids on contained resources +* Validator: fix bug looking up code system +* Validator: Look for cs on other server in missed location +* Validator: fix bug accessing code system from other terminology server if no version specified +* Validator: validate displaylanguage when validating codes +* Renderer: Possible fix for an NPE reported bu a user with no context +* QA: fix bug parsing `script` tag in xhtml - handling `<` characters +* QA: Tighted up handling of errors when HTML pages can't be parsed +* Publication Process: remove FTP support in -go-publish (it's all git now) +* Publication Process: radd support for creation phase diff --git a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/HTMLInspector.java b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/HTMLInspector.java index cc59e14f8..74398a312 100644 --- a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/HTMLInspector.java +++ b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/HTMLInspector.java @@ -511,7 +511,8 @@ private void loadFile(String s, String base, List messages) { } catch (FHIRFormatError | IOException e) { x = null; if (htmlName || !(e.getMessage().startsWith("Unable to Parse HTML - does not start with tag.") || e.getMessage().startsWith("Malformed XHTML"))) { - messages.add(new ValidationMessage(Source.LinkChecker, IssueType.STRUCTURE, s, e.getMessage(), IssueSeverity.ERROR).setLocationLink(makeLocal(f.getAbsolutePath()))); + messages.add(new ValidationMessage(Source.LinkChecker, IssueType.STRUCTURE, s, e.getMessage(), IssueSeverity.ERROR).setLocationLink(makeLocal(f.getAbsolutePath())).setMessageId("HTML_PARSING_FAILED")); + missingPublishBox = true; } } if (x != null) { diff --git a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/ValidationServices.java b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/ValidationServices.java index be7a9d870..70f4b255f 100644 --- a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/ValidationServices.java +++ b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/ValidationServices.java @@ -64,6 +64,7 @@ import org.hl7.fhir.r5.utils.validation.IResourceValidator; import org.hl7.fhir.r5.utils.validation.IValidationPolicyAdvisor; import org.hl7.fhir.r5.utils.validation.IValidatorResourceFetcher; +import org.hl7.fhir.r5.utils.validation.IValidationPolicyAdvisor.SpecialValidationAction; import org.hl7.fhir.r5.utils.validation.constants.BindingKind; import org.hl7.fhir.r5.utils.validation.constants.ContainedReferenceValidationPolicy; import org.hl7.fhir.r5.utils.validation.constants.ReferenceValidationPolicy; @@ -492,4 +493,10 @@ public IValidationPolicyAdvisor getPolicyAdvisor() { public IValidationPolicyAdvisor setPolicyAdvisor(IValidationPolicyAdvisor policyAdvisor) { throw new Error("Not supported"); // ! } + + @Override + public SpecialValidationAction policyForSpecialValidation(IResourceValidator validator, Object appContext, SpecialValidationRule rule, String stackPath, Element resource, Element element) { + return SpecialValidationAction.CHECK_RULE; + + } } diff --git a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/utils/HL7OrgFhirFixer.java b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/utils/HL7OrgFhirFixer.java new file mode 100644 index 000000000..1671741a9 --- /dev/null +++ b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/utils/HL7OrgFhirFixer.java @@ -0,0 +1,164 @@ +package org.hl7.fhir.igtools.publisher.utils; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.hl7.fhir.utilities.TextFile; +import org.hl7.fhir.utilities.Utilities; +import org.hl7.fhir.utilities.json.model.JsonElement; +import org.hl7.fhir.utilities.json.model.JsonObject; +import org.hl7.fhir.utilities.json.parser.JsonParser; + + +public class HL7OrgFhirFixer { + + public static void main(String[] args) throws IOException { + File folder = new File("/Users/grahamegrieve/web/www.hl7.org.fhir"); + new HL7OrgFhirFixer().execute(folder.getAbsolutePath().length(), folder, true); + } + + private void execute(int rootLen, File folder, boolean root) throws IOException { + boolean isSpec = isSpec(folder); + if (isSpec) { + System.out.print("Found Spec at "+folder.getAbsolutePath()); + processSpec(rootLen, folder); + } + boolean isEmpty = true; + for (File f : folder.listFiles()) { + if (f.isDirectory()) { + isEmpty = false; + execute(rootLen, f, false); + } else if (f.getName().endsWith(".asp")) { + f.delete(); + } else { + isEmpty = false; + } + } + if (isEmpty) { + folder.delete(); + } + } + + private boolean isSpec(File folder) throws IOException { + if (new File(Utilities.path(folder.getAbsolutePath(), "package.tgz")).exists()) { + return true; + } + if (new File(Utilities.path(folder.getAbsolutePath(), "patient.html")).exists()) { + return true; + } + if (new File(Utilities.path(folder.getAbsolutePath(), "patient.htm")).exists()) { + return true; + } + return false; + } + + private void processSpec(int rootLen, File folder) throws IOException { + Map> list = new HashMap<>(); + + scanFolder(rootLen, folder, list); + + int size = 0; + for (String t : list.keySet()) { + size += list.get(t).size(); + String tf = Utilities.noString(t) ? folder.getAbsolutePath() : Utilities.path(folder.getAbsolutePath(), t); + Utilities.createDirectoryNC(tf); + Map map = list.get(t); + for (String id : map.keySet()) { + String idf = Utilities.path(tf, id); + Utilities.createDirectoryNC(idf); + String rf = Utilities.path(idf, "index.php"); + TextFile.stringToFile(genRedirect(map.get(id)), rf); + } + } + System.out.println(" "+size+" resources"); + } + + private String genRedirect(String url) { + String ub = url.replace(".json", ""); + return "\r\n"+ +"\r\n"+ +"You should not be seeing this page. If you do, PHP has failed badly.\r\n"+ +"\r\n"; + } + + private void scanFolder(int rootLen, File folder, Map> list) throws IOException { + for (File f : folder.listFiles()) { + if (f.getName().equals("web.config")) { + f.delete(); + } else if (f.getName().endsWith(".asp")) { + f.delete(); + } else if (f.isDirectory()) { + if (!isSpec(folder)) { + scanFolder(rootLen, f, list); + } + } else if (f.getName().endsWith(".json") && !f.getName().contains(".canonical.")) { + try { + JsonElement je = JsonParser.parse(f); + if (je.isJsonObject()) { + JsonObject j = je.asJsonObject(); + if (j.has("resourceType") && j.has("id")) { + String rt = j.asString("resourceType"); + String id = j.asString("id"); + String link = f.getAbsolutePath().substring(rootLen); + Map ft = makeMapForType(list, rt); + ft.put(id, link); + if (j.has("url")) { + String url = j.asString("url"); + if (url.startsWith("http://hl7.org/fhir/")) { + String tail = url.substring(20); + if (j.has("version")) { + ft.put(id+"|"+j.asString("version"), link); + } + if (!tail.contains("/") && !tail.contains(".")) { + ft = makeMapForType(list, ""); + ft.put(tail, link); + if (j.has("version")) { + ft.put(tail+"|"+j.asString("version"), link); + } + } + } + } + } + } + } catch (Exception e) { + } + } + } + } + + public Map makeMapForType(Map> list, String rt) { + Map ft = list.get(rt); + if (ft == null) { + ft = new HashMap(); + list.put(rt, ft); + } + return ft; + } +} \ No newline at end of file diff --git a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/utils/HL7OrgFhirFixerForExtensions.java b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/utils/HL7OrgFhirFixerForExtensions.java new file mode 100644 index 000000000..0e1485c80 --- /dev/null +++ b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/utils/HL7OrgFhirFixerForExtensions.java @@ -0,0 +1,107 @@ +package org.hl7.fhir.igtools.publisher.utils; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.hl7.fhir.r5.model.CanonicalResource; +import org.hl7.fhir.r5.model.Resource; +import org.hl7.fhir.utilities.TextFile; +import org.hl7.fhir.utilities.Utilities; + +public class HL7OrgFhirFixerForExtensions { + + public static void main(String[] args) throws IOException { + File folderRoot = new File("/Users/grahamegrieve/web/www.hl7.org.fhir"); + File folderExt = new File("/Users/grahamegrieve/web/www.hl7.org.fhir/extensions"); + new HL7OrgFhirFixerForExtensions().execute(folderRoot, folderExt); + } + + private void execute(File folderRoot, File folderExt) throws IOException { + + scanForEmptyFolders(folderRoot); + } + + private void scanForEmptyFolders(File ff) throws IOException { + boolean indexed = false; + for (File f : ff.listFiles()) { + if (f.isDirectory()) { + if (!Utilities.existsInList(f.getName(), "assets", "assets-hist", "dist", "dist-hist", "less", "images", "less-glyphicons", "less-joyo", "html", "css", + "quick", "qicore", "external", "examples", "sid", ".git", "ehrsrle")) { + scanForEmptyFolders(f); + } + } else { + if (Utilities.existsInList(f.getName(), "index.html", "index.php", "web.config", ".no.index")) { + indexed = true; + } + if (f.getName().equals(".no.index")) { + f.delete(); + } + if (f.getName().equals("web.config")) { + String s = TextFile.fileToString(f); + String url = s.substring(s.indexOf("destination=")+13); + url = url.substring(0, url.indexOf("\"")); + String rf = genRedirect(url.replace("http://hl7.org/fhir", "")); + String dst = Utilities.path(ff, "index.php"); + TextFile.stringToFile(rf, dst); + f.delete(); + } + } + } + } + + private boolean lowerCaseHtmlExists(File ff) throws IOException { + String path = Utilities.path(ff.getParentFile(), ff.getName().toLowerCase()+".html"); + return new File(path).exists(); + } + + private String genRedirect(String url) { + return "\r\n"+ +"\r\n"+ +"{ \"message\" : \"not-found\" }\r\n"+ +"\r\n"; +} + + private boolean isResourceName(String name) { + return Utilities.existsInList(name, + "Account", "ActivityDefinition", "ActorDefinition", "AdministrableProductDefinition", "AdverseEvent", "AllergyIntolerance", "Appointment", "AppointmentResponse", "ArtifactAssessment", "AuditEvent", + "Basic", "Binary", "BiologicallyDerivedProduct", "BiologicallyDerivedProductDispense", "BodySite", "BodyStructure", "Bundle", "CanonicalResource", "CapabilityStatement", "CarePlan", "CareTeam", + "CatalogEntry", "ChargeItem", "ChargeItemDefinition", "Citation", "Claim", "ClaimResponse", "ClinicalImpression", "ClinicalUseDefinition", "CodeSystem", "Communication", "CommunicationRequest", + "CompartmentDefinition", "Composition", "ConceptMap", "Condition", "ConditionDefinition", "Conformance", "Consent", "Contract", "Coverage", "CoverageEligibilityRequest", "CoverageEligibilityResponse", + "DataElement", "DetectedIssue", "Device", "DeviceAssociation", "DeviceComponent", "DeviceDefinition", "DeviceDispense", "DeviceMetric", "DeviceRequest", "DeviceUsage", "DeviceUseRequest", + "DeviceUseStatement", "DiagnosticOrder", "DiagnosticReport", "DocumentManifest", "DocumentReference", "DomainResource", "EffectEvidenceSynthesis", "EligibilityRequest", "EligibilityResponse", + "Encounter", "EncounterHistory", "Endpoint", "EnrollmentRequest", "EnrollmentResponse", "EpisodeOfCare", "EventDefinition", "Evidence", "EvidenceReport", "EvidenceVariable", "ExampleScenario", + "ExpansionProfile", "ExplanationOfBenefit", "FamilyMemberHistory", "Flag", "FormularyItem", "GenomicStudy", "Goal", "GraphDefinition", "Group", "GuidanceResponse", "HealthcareService", "ImagingManifest", + "ImagingObjectSelection", "ImagingSelection", "ImagingStudy", "Immunization", "ImmunizationEvaluation", "ImmunizationRecommendation", "ImplementationGuide", "Ingredient", "InsurancePlan", "InventoryItem", + "InventoryReport", "Invoice", "Library", "Linkage", "List", "Location", "ManufacturedItemDefinition", "Measure", "MeasureReport", "Media", "Medication", "MedicationAdministration", "MedicationDispense", + "MedicationKnowledge", "MedicationOrder", "MedicationPrescription", "MedicationRequest", "MedicationStatement", "MedicationUsage", "MedicinalProduct", "MedicinalProductAuthorization", "MedicinalProductContraindication", + "MedicinalProductDefinition", "MedicinalProductIndication", "MedicinalProductIngredient", "MedicinalProductInteraction", "MedicinalProductManufactured", "MedicinalProductPackaged", + "MedicinalProductPharmaceutical", "MedicinalProductUndesirableEffect", "MessageDefinition", "MessageHeader", "MetadataResource", "MolecularSequence", "NamingSystem", "NutritionIntake", "NutritionOrder", + "NutritionProduct", "Observation", "ObservationDefinition", "OperationDefinition", "OperationOutcome", "Order", "OrderResponse", "Organization", "OrganizationAffiliation", "PackagedProductDefinition", + "Parameters", "Patient", "PaymentNotice", "PaymentReconciliation", "Permission", "Person", "PlanDefinition", "Practitioner", "PractitionerRole", "Procedure", "ProcedureRequest", "ProcessRequest", + "ProcessResponse", "Provenance", "Questionnaire", "QuestionnaireResponse", "ReferralRequest", "RegulatedAuthorization", "RelatedPerson", "RequestGroup", "RequestOrchestration", "Requirements", + "ResearchDefinition", "ResearchElementDefinition", "ResearchStudy", "ResearchSubject", "Resource", "RiskAssessment", "RiskEvidenceSynthesis", "Schedule", "SearchParameter", "Sequence", "ServiceDefinition", + "ServiceRequest", "Slot", "Specimen", "SpecimenDefinition", "StructureDefinition", "StructureMap", "Subscription", "SubscriptionStatus", "SubscriptionTopic", "Substance", "SubstanceDefinition", + "SubstanceNucleicAcid", "SubstancePolymer", "SubstanceProtein", "SubstanceReferenceInformation", "SubstanceSourceMaterial", "SubstanceSpecification", "SupplyDelivery", "SupplyRequest", "Task", + "TerminologyCapabilities", "TestPlan", "TestReport", "TestScript", "Transport", "ValueSet", "VerificationResult", "VisionPrescription", + + "Supply", "Contraindication", "CapabilityStatement2", "ClinicalUseIssue", "DiagnosticRequest", "NutritionRequest", + "DecisionSupportServiceModule", "ModuleDefinition", "GuidanceRequest", "DecisionSupportRule", "ModuleMetadata", "OrderSet", "ModuleDefinition", "GuidanceRequest", "EvidenceFocus","ItemInstance" + ); + } + + +} \ No newline at end of file diff --git a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/renderers/ValidationPresenter.java b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/renderers/ValidationPresenter.java index 1944ff838..d75346eff 100644 --- a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/renderers/ValidationPresenter.java +++ b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/renderers/ValidationPresenter.java @@ -981,7 +981,7 @@ public static List filterMessages(List mes Set msgs = new HashSet<>(); for (ValidationMessage message : messages) { boolean passesFilter = true; - if (canSuppressErrors || !message.getLevel().isError()) { + if ((canSuppressErrors || !message.getLevel().isError()) && (!"HTML_PARSING_FAILED".equals(message.getMessageId()))) { if (suppressedMessages.contains(message.getDisplay(), message) || suppressedMessages.contains(message.getMessage(), message) || suppressedMessages.contains(message.getHtml(), message) || suppressedMessages.contains(message.getMessageId(), message) || suppressedMessages.contains(message.getInvId(), message)) { diff --git a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/web/PublicationProcess.java b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/web/PublicationProcess.java index 19c803ec1..0affd5989 100644 --- a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/web/PublicationProcess.java +++ b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/web/PublicationProcess.java @@ -67,6 +67,7 @@ public class PublicationProcess { */ public enum PublicationProcessMode { + CREATION, WORKING, MILESTONE, TECHNICAL_CORRECTION; @@ -82,6 +83,9 @@ public static PublicationProcessMode fromCode(String s) { if ("milestone".equals(s)) { return MILESTONE; } + if ("creation".equals(s)) { + return CREATION; + } if ("technical-correction".equals(s)) { return TECHNICAL_CORRECTION; } @@ -120,8 +124,7 @@ public void publish(String source, String web, String date, String registrySourc System.out.println(s); System.out.println("---------------"); System.out.println("Publication Run: publish "+source+" to "+web); - boolean manualCheck = "true".equals(getNamedParam(args, "-manual-check")); - List res = publishInner(source, web, date, registrySource, history, templatesSrc, temp, logger, args, manualCheck); + List res = publishInner(source, web, date, registrySource, history, templatesSrc, temp, logger, args); if (res.size() == 0) { System.out.println("Success"); } else { @@ -150,7 +153,7 @@ private static String toMB(long maxMemory) { return Long.toString(maxMemory / (1024*1024)); } - public List publishInner(String source, String web, String date, String registrySource, String history, String templateSrc, String temp, PublisherConsoleLogger logger, String[] args, boolean manualCheck) throws Exception { + public List publishInner(String source, String web, String date, String registrySource, String history, String templateSrc, String temp, PublisherConsoleLogger logger, String[] args) throws Exception { List res = new ArrayList<>(); if (temp == null) { @@ -168,7 +171,7 @@ public List publishInner(String source, String web, String da File fRoot = checkDirectory(workingRoot, res, "Working Web Folder"); WebSourceProvider src = new WebSourceProvider(workingRoot, web); if (getNamedParam(args, "-upload-server") != null) { - src.configureUpload(getNamedParam(args, "-upload-server"), getNamedParam(args, "-upload-path"), getNamedParam(args, "-upload-user"), getNamedParam(args, "-upload-password")); + throw new Error("Uploading files by FTP is no longer supported"); } checkDirectory(Utilities.path(workingRoot, "ig-build-zips"), res, "Destination Zip Folder", true); @@ -215,17 +218,6 @@ public List publishInner(String source, String web, String da if (!check(res, version != null, "Source Package has no version")) { return res; } - JsonObject json = JsonParser.parseObject(npm.load("package", "package.json")); - JsonObject dep = json.getJsonObject("dependencies"); - if (dep != null) { - for (JsonProperty jp : dep.getProperties()) { - String ver = jp.getValue().asJsonString().getValue(); - if ("current".equals(ver) || "dev".equals(ver)) { - check(res, false, "Package "+json.asString("name")+"#"+json.asString("version")+" depends on "+jp.getName()+"#"+ver+" which is not allowed (current version check)"); - return res; - } - } - } // --- Rules for layout depend on publisher ------ boolean help = true; @@ -241,7 +233,7 @@ public List publishInner(String source, String web, String da String destination = Utilities.path(workingRoot, calcByRule(rules.str("destination"), p)); help = false; publishInner2(source, web, date, registrySource, history, templateSrc, temp, logger, args, - manualCheck, destination, workingRoot, res, src, id, canonical, version, npm, pubSetup, qa, + destination, workingRoot, res, src, id, canonical, version, npm, pubSetup, qa, fSource, fOutput, fRoot, fRegistry, fHistory, runNumber); } } @@ -286,7 +278,7 @@ private String calcByRule(String str, String[] p) { public List publishInner2(String source, String web, String date, String registrySource, String history, String templateSrc, String temp, - PublisherConsoleLogger logger, String[] args, boolean manualCheck, String destination, String workingRoot, List res, WebSourceProvider src, + PublisherConsoleLogger logger, String[] args, String destination, String workingRoot, List res, WebSourceProvider src, String id, String canonical, String version, NpmPackage npm, JsonObject pubSetup, JsonObject qa, File fSource, File fOutput, File fRoot, File fRegistry, File fHistory, int runNumber) throws Exception { System.out.println("Relative directory for IG is '"+destination.substring(workingRoot.length())+"'"); @@ -308,6 +300,20 @@ public List publishInner2(String source, String web, String d JsonObject prSrc = JsonParser.parseObject(loadFile("Source publication request", Utilities.path(source, "publication-request.json"))); PublicationProcessMode mode = PublicationProcessMode.fromCode(prSrc.asString("mode")); + + if (mode != PublicationProcessMode.CREATION) { + JsonObject json = JsonParser.parseObject(npm.load("package", "package.json")); + JsonObject dep = json.getJsonObject("dependencies"); + if (dep != null) { + for (JsonProperty jp : dep.getProperties()) { + String ver = jp.getValue().asJsonString().getValue(); + if ("current".equals(ver) || "dev".equals(ver)) { + check(res, false, "Package "+json.asString("name")+"#"+json.asString("version")+" depends on "+jp.getName()+"#"+ver+" which is not allowed (current version check)"); + return res; + } + } + } + } boolean first = prSrc.asBoolean("first"); src.needOptionalFile(Utilities.noString(relDest) ? "package-list.json" : Utilities.path(relDest,"package-list.json")); if (first) { @@ -403,7 +409,7 @@ public List publishInner2(String source, String web, String d // well, we've run out of things to test... time to actually try... if (res.size() == 0) { doPublish(fSource, fOutput, qa, destination, destVer, pathVer, fRoot, pubSetup, pl, prSrc, fRegistry, npm, mode, date, fHistory, temp, logger, - pubSetup.getJsonObject("website").asString("url"), src, sft, relDest, templateSrc, first, indexes, Calendar.getInstance(), getComputerName(), IGVersionUtil.getVersionString(), gitSrcId(source), Integer.toString(runNumber), tcName, manualCheck, + pubSetup.getJsonObject("website").asString("url"), src, sft, relDest, templateSrc, first, indexes, Calendar.getInstance(), getComputerName(), IGVersionUtil.getVersionString(), gitSrcId(source), Integer.toString(runNumber), tcName, workingRoot); } return res; @@ -476,133 +482,116 @@ private InputStream loadFile(String string, String path) throws FileNotFoundExce private void doPublish(File fSource, File fOutput, JsonObject qa, String destination, String destVer, String pathVer, File fRoot, JsonObject pubSetup, PackageList pl, JsonObject prSrc, File fRegistry, NpmPackage npm, PublicationProcessMode mode, String date, File history, String tempDir, PublisherConsoleLogger logger, String url, WebSourceProvider src, File sft, String relDest, String templateSrc, boolean first, Map indexes, - Calendar genDate, String username, String version, String gitSrcId, String runNumber, String tcName, boolean manualCheck, String workingRoot) throws Exception { + Calendar genDate, String username, String version, String gitSrcId, String runNumber, String tcName, String workingRoot) throws Exception { // ok. all our tests have passed. // 1. do the publication build(s) - System.out.println("All checks passed. Do the publication build from "+fSource.getAbsolutePath()+" and publish to "+destination); + List existingFiles = new ArrayList<>(); + if (mode == PublicationProcessMode.CREATION) { + System.out.println("All checks passed. Create an empty history at "+destination); + } else { + System.out.println("All checks passed. Do the publication build from "+fSource.getAbsolutePath()+" and publish to "+destination); - // create a temporary copy and build in that: - File temp = cloneToTemp(tempDir, fSource, npm.name()+"#"+npm.version()); - File tempM = null; - System.out.println("Build IG at "+fSource.getAbsolutePath()+": final copy suitable for publication (in "+temp.getAbsolutePath()+")"); - runBuild(qa, temp.getAbsolutePath(), new String[] {"-ig", temp.getAbsolutePath(), "-resetTx", "-publish", pathVer, "-no-exit"}); + // create a temporary copy and build in that: + File temp = cloneToTemp(tempDir, fSource, npm.name()+"#"+npm.version()); + File tempM = null; + System.out.println("Build IG at "+fSource.getAbsolutePath()+": final copy suitable for publication (in "+temp.getAbsolutePath()+")"); + runBuild(qa, temp.getAbsolutePath(), new String[] {"-ig", temp.getAbsolutePath(), "-resetTx", "-publish", pathVer, "-no-exit"}); - if (mode != PublicationProcessMode.WORKING) { - tempM = cloneToTemp(tempDir, temp, npm.name()+"#"+npm.version()+"-milestone"); - System.out.println("Build IG at "+fSource.getAbsolutePath()+": final copy suitable for publication (in "+tempM.getAbsolutePath()+") (milestone build)"); - runBuild(qa, tempM.getAbsolutePath(), new String[] {"-ig", tempM.getAbsolutePath(), "-publish", pathVer, "-milestone", "-no-exit"}); - } + if (mode != PublicationProcessMode.WORKING) { + tempM = cloneToTemp(tempDir, temp, npm.name()+"#"+npm.version()+"-milestone"); + System.out.println("Build IG at "+fSource.getAbsolutePath()+": final copy suitable for publication (in "+tempM.getAbsolutePath()+") (milestone build)"); + runBuild(qa, tempM.getAbsolutePath(), new String[] {"-ig", tempM.getAbsolutePath(), "-publish", pathVer, "-milestone", "-no-exit"}); + } - // 2. make a copy of what we built - System.out.println("Keep a copy of the build directory at "+Utilities.path(fRoot.getAbsolutePath(), "ig-build-zips", npm.name()+"#"+npm.version()+".zip")); + // 2. make a copy of what we built + System.out.println("Keep a copy of the build directory at "+Utilities.path(fRoot.getAbsolutePath(), "ig-build-zips", npm.name()+"#"+npm.version()+".zip")); - // 2.1. Delete the ".git" subfolder - File gitFolder = new File(temp, ".git"); - if (gitFolder.exists()) { + // 2.1. Delete the ".git" subfolder + File gitFolder = new File(temp, ".git"); + if (gitFolder.exists()) { FileUtils.deleteDirectory(gitFolder); - } + } - zipFolder(temp, Utilities.path(fRoot.getAbsolutePath(), "ig-build-zips", npm.name()+"#"+npm.version()+".zip")); + zipFolder(temp, Utilities.path(fRoot.getAbsolutePath(), "ig-build-zips", npm.name()+"#"+npm.version()+".zip")); - System.out.println(""); - System.out.println("ok. All checks passed. Publish v"+npm.version()+" to "+destVer); + System.out.println(""); + System.out.println("ok. All checks passed. Publish v"+npm.version()+" to "+destVer); - // 3. create the folder {root}/{realm}/{code}/{subdir} - System.out.println("Copy the IG to "+destVer); - Utilities.createDirectory(destVer); - FileUtils.copyDirectory(new File(Utilities.path(temp.getAbsolutePath(), "output")), new File(destVer)); - List subPackages = loadSubPackageList(Utilities.path(temp.getAbsolutePath(), "output", "sub-package-list.json")); + // 3. create the folder {root}/{realm}/{code}/{subdir} + System.out.println("Copy the IG to "+destVer); + Utilities.createDirectory(destVer); + FileUtils.copyDirectory(new File(Utilities.path(temp.getAbsolutePath(), "output")), new File(destVer)); + List subPackages = loadSubPackageList(Utilities.path(temp.getAbsolutePath(), "output", "sub-package-list.json")); + // now, update the package list + System.out.println("Update "+Utilities.path(destination, "package-list.json")); + PackageListEntry plVer = updatePackageList(pl, fSource.getAbsolutePath(), prSrc, pathVer, Utilities.path(destination, "package-list.json"), mode, date, npm.fhirVersion(), Utilities.pathURL(pubSetup.asString("url"), tcName), subPackages); + updatePublishBox(pl, plVer, destVer, pathVer, destination, fRoot.getAbsolutePath(), false, ServerType.fromCode(pubSetup.getJsonObject("website").asString("server")), sft, null, url); - // now, update the package list - System.out.println("Update "+Utilities.path(destination, "package-list.json")); - PackageListEntry plVer = updatePackageList(pl, fSource.getAbsolutePath(), prSrc, pathVer, Utilities.path(destination, "package-list.json"), mode, date, npm.fhirVersion(), Utilities.pathURL(pubSetup.asString("url"), tcName), subPackages); - updatePublishBox(pl, plVer, destVer, pathVer, destination, fRoot.getAbsolutePath(), false, ServerType.fromCode(pubSetup.getJsonObject("website").asString("server")), sft, null, url); - - List existingFiles = new ArrayList<>(); - if (mode != PublicationProcessMode.WORKING) { - String igSrc = Utilities.path(tempM.getAbsolutePath(), "output"); - if (mode == PublicationProcessMode.TECHNICAL_CORRECTION) { - System.out.println("This is a technical correction - publish v"+npm.version()+" to "+destination+" but archive first"); - produceArchive(destVer, Utilities.path(igSrc,tcName)); - } else { - System.out.println("This is a milestone release - publish v"+npm.version()+" to "+destination); - } - - List ignoreList = new ArrayList<>(); - ignoreList.add(destVer); - // get the current content from the source - for (PackageListEntry v : pl.versions()) { - if (v != plVer) { - String path = v.determineLocalPath(url, fRoot.getAbsolutePath()); - if (path != null) { - String relPath = Utilities.getRelativePath(fRoot.getAbsolutePath(), path); - ignoreList.add(path); - src.needFolder(relPath, false); + + + if (mode != PublicationProcessMode.WORKING) { + String igSrc = Utilities.path(tempM.getAbsolutePath(), "output"); + if (mode == PublicationProcessMode.TECHNICAL_CORRECTION) { + System.out.println("This is a technical correction - publish v"+npm.version()+" to "+destination+" but archive first"); + produceArchive(destVer, Utilities.path(igSrc,tcName)); + } else { + System.out.println("This is a milestone release - publish v"+npm.version()+" to "+destination); + } + + List ignoreList = new ArrayList<>(); + ignoreList.add(destVer); + // get the current content from the source + for (PackageListEntry v : pl.versions()) { + if (v != plVer) { + String path = v.determineLocalPath(url, fRoot.getAbsolutePath()); + if (path != null) { + String relPath = Utilities.getRelativePath(fRoot.getAbsolutePath(), path); + ignoreList.add(path); + src.needFolder(relPath, false); + } } } + // we do this first in the output so we can get a proper diff + updatePublishBox(pl, plVer, igSrc, pathVer, igSrc, fRoot.getAbsolutePath(), true, ServerType.fromCode(pubSetup.getJsonObject("website").asString("server")), sft, null, url); + + System.out.println("Check for Files to delete"); + List newFiles = Utilities.listAllFiles(igSrc, null); + List historyFiles = Utilities.listAllFiles(history.getAbsolutePath(), null); + existingFiles = Utilities.listAllFiles(destination, ignoreList); + existingFiles.removeAll(newFiles); + existingFiles.removeAll(historyFiles); + existingFiles.remove("package-list.json"); + existingFiles.removeIf(s -> s.endsWith("web.config")); + for (String s : existingFiles) { + new File(Utilities.path(destination, s)).delete(); + } + System.out.println(" ... "+existingFiles.size()+" files"); + System.out.println("Copy to new IG to "+destination); + FileUtils.copyDirectory(new File(Utilities.path(tempM.getAbsolutePath(), "output")), new File(destination)); + } else { + src.cleanFolder(relDest); } - // we do this first in the output so we can get a proper diff - updatePublishBox(pl, plVer, igSrc, pathVer, igSrc, fRoot.getAbsolutePath(), true, ServerType.fromCode(pubSetup.getJsonObject("website").asString("server")), sft, null, url); - - System.out.println("Check for Files to delete"); - List newFiles = Utilities.listAllFiles(igSrc, null); - List historyFiles = Utilities.listAllFiles(history.getAbsolutePath(), null); - existingFiles = Utilities.listAllFiles(destination, ignoreList); - existingFiles.removeAll(newFiles); - existingFiles.removeAll(historyFiles); - existingFiles.remove("package-list.json"); - existingFiles.removeIf(s -> s.endsWith("web.config")); - for (String s : existingFiles) { - new File(Utilities.path(destination, s)).delete(); - } - System.out.println(" ... "+existingFiles.size()+" files"); - System.out.println("Copy to new IG to "+destination); - FileUtils.copyDirectory(new File(Utilities.path(tempM.getAbsolutePath(), "output")), new File(destination)); - if (src.isWeb()) { - new WebSiteArchiveBuilder().buildArchives(new File(destination), fRoot.getAbsolutePath(), url); - } - } else { - if (src.isWeb()) { - new WebSiteArchiveBuilder().buildArchive(destVer, new ArrayList<>()); + NpmPackage npmB = NpmPackage.fromPackage(new FileInputStream(Utilities.path(destVer, "package.tgz"))); + updateFeed(fRoot, destVer, pl, plVer, pubSetup.forceObject("feeds").asString("package"), false, src, pubSetup.forceObject("website").asString("org"), npmB, genDateS(genDate), username, version, gitSrcId, runNumber); + updateFeed(fRoot, destVer, pl, plVer, pubSetup.forceObject("feeds").asString("publication"), true, src, pubSetup.forceObject("website").asString("org"), npmB, genDateS(genDate), username, version, gitSrcId, runNumber); + new PackageRegistryBuilder(workingRoot).update(destination.substring(workingRoot.length()+1), pl); + IndexMaintainer ndx = getIndexForIg(indexes, npmB.id()); + if (ndx != null) { + src.needFile(Utilities.changeFileExt(ndx.path(), ".json")); + ndx.loadJson(); + ndx.updateForPublication(pl, plVer, mode != PublicationProcessMode.WORKING); + ndx.buildJson(); + ndx.execute(); } - src.cleanFolder(relDest); + updateRegistry(fRegistry, pl, plVer, npmB); + System.out.println("Build is complete. "+src.verb()+" from "+ fRoot.getAbsolutePath()); } - NpmPackage npmB = NpmPackage.fromPackage(new FileInputStream(Utilities.path(destVer, "package.tgz"))); - updateFeed(fRoot, destVer, pl, plVer, pubSetup.forceObject("feeds").asString("package"), false, src, pubSetup.forceObject("website").asString("org"), npmB, genDateS(genDate), username, version, gitSrcId, runNumber); - updateFeed(fRoot, destVer, pl, plVer, pubSetup.forceObject("feeds").asString("publication"), true, src, pubSetup.forceObject("website").asString("org"), npmB, genDateS(genDate), username, version, gitSrcId, runNumber); - new PackageRegistryBuilder(workingRoot).update(destination.substring(workingRoot.length()+1), pl); - new HistoryPageUpdater().updateHistoryPage(history.getAbsolutePath(), destination, templateSrc, !first); - - IndexMaintainer ndx = getIndexForIg(indexes, npmB.id()); - if (ndx != null) { - src.needFile(Utilities.changeFileExt(ndx.path(), ".json")); - ndx.loadJson(); - ndx.updateForPublication(pl, plVer, mode != PublicationProcessMode.WORKING); - ndx.buildJson(); - ndx.execute(); - } - boolean go; - if (manualCheck) { - System.out.println("The build is complete, and ready to be applied. Check the output at "+fRoot.getAbsolutePath()); - System.out.println("Do you wish to continue? (Y/N)"); - int r = System.in.read(); - go = r == 'y' || r == 'Y'; - } else { - System.out.println("Build is complete. "+src.verb()+" from "+ fRoot.getAbsolutePath()); - go = true; - } - if (go) { - updateRegistry(fRegistry, pl, plVer, npmB); - logger.stop(); - FileUtils.copyFile(new File(logger.getFilename()), new File(Utilities.path(fRoot.getAbsolutePath(), "ig-build-zips", npm.name()+"#"+npm.version()+".log"))); - src.finish(relDest, existingFiles); - System.out.println("Finished Publishing. "+src.instructions(existingFiles.size())); - } else { - System.out.println("No!"); - System.out.print("Changes not applied. Finished"); - } + logger.stop(); + FileUtils.copyFile(new File(logger.getFilename()), new File(Utilities.path(fRoot.getAbsolutePath(), "ig-build-zips", npm.name()+"#"+npm.version()+".log"))); + src.finish(relDest, existingFiles); + System.out.println("Finished Publishing. "+src.instructions(existingFiles.size())); exitCode = 0; } diff --git a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/web/WebSourceProvider.java b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/web/WebSourceProvider.java index 1aabaf0e0..f3a48bed1 100644 --- a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/web/WebSourceProvider.java +++ b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/web/WebSourceProvider.java @@ -53,27 +53,12 @@ public int compare(String o1, String o2) { private String destination; private String source; - private boolean web; private Map folderSources = new HashMap<>(); - private boolean upload = false; - private String uploadServer; - private String uploadPath; - private String uploadUser; - private String uploadPword; public WebSourceProvider(String destination, String source) { super(); this.destination = destination; this.source = source; - web = Utilities.startsWithInList(source, "http://", "https://"); - } - - public void configureUpload(String server, String path, String user, String pword) { - upload = true; - uploadServer = server; - uploadPath = path; - uploadUser = user; - uploadPword = pword; } public void needFile(String path) throws IOException { @@ -82,21 +67,11 @@ public void needFile(String path) throws IOException { if (df.exists()) { df.delete(); } - if (web) { - String url = Utilities.pathURL(source, path)+"?nocache=" + System.currentTimeMillis(); - System.out.println("Fetch "+ url); - long t = System.currentTimeMillis(); - HTTPResult res = ManagedWebAccess.get(Arrays.asList("web"), url); - res.checkThrowException(); - TextFile.bytesToFile(res.getContent(), df); - System.out.println(" ... done ("+stats(res.getContent().length, t)+")"); - } else { - File sf = new File(Utilities.path(source, path)); - if (!sf.exists()) { - throw new Error("Error: Attempt to copy "+sf.getAbsolutePath()+" but it doesn't exist"); - } - Utilities.copyFile(sf, df); + File sf = new File(Utilities.path(source, path)); + if (!sf.exists()) { + throw new Error("Error: Attempt to copy "+sf.getAbsolutePath()+" but it doesn't exist"); } + Utilities.copyFile(sf, df); } public void needFolder(String path, boolean clear) throws IOException { @@ -109,25 +84,16 @@ public void needFolder(String path, boolean clear) throws IOException { } else if (clear) { Utilities.clearDirectory(df.getAbsolutePath()); } - if (web) { - String url = Utilities.pathURL(source, path, "_ig-pub-archive.zip")+"?nocache=" + System.currentTimeMillis(); - System.out.println("Fetch "+ url); - long t = System.currentTimeMillis(); - HTTPResult res = ManagedWebAccess.get(Arrays.asList("web"), url); - res.checkThrowException(); - folderSources.put(path, res.getContent()); - Utilities.unzip(new ByteArrayInputStream(res.getContent()), df.getAbsolutePath()); - System.out.println(" ... done ("+stats(res.getContent().length, t)+")"); - } else { - File sf = new File(Utilities.path(source, path)); - if (!sf.exists()) { - throw new Error("Error: Attempt to copy from "+sf.getAbsolutePath()+" but it doesn't exist"); - } - if (!sf.isDirectory()) { - throw new Error("Error: Attempt to copy from "+sf.getAbsolutePath()+" but it isn't a folder"); - } - Utilities.copyDirectory(sf.getAbsolutePath(), df.getAbsolutePath(), null); + + File sf = new File(Utilities.path(source, path)); + if (!sf.exists()) { + throw new Error("Error: Attempt to copy from "+sf.getAbsolutePath()+" but it doesn't exist"); + } + if (!sf.isDirectory()) { + throw new Error("Error: Attempt to copy from "+sf.getAbsolutePath()+" but it isn't a folder"); } + Utilities.copyDirectory(sf.getAbsolutePath(), df.getAbsolutePath(), null); + } private String stats(long length, long t) { @@ -142,17 +108,7 @@ private String stats(long length, long t) { public void cleanFolder(String path) throws IOException { File df = new File(path == null ? destination : Utilities.path(destination, path)); - if (web) { - Path target = Path.of(df.getAbsolutePath()); - byte[] buf = folderSources.get(path); - if (buf != null) { - ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(buf); - cleanZipTargets(target, byteArrayInputStream); - Utilities.deleteEmptyFolders(df); - } - } else { - // do nothing? - } + // do nothing? } protected static void cleanZipTargets(Path target, InputStream inputStream) throws IOException { @@ -174,121 +130,20 @@ public void needOptionalFile(String path) throws IOException { if (df.exists()) { df.delete(); } - if (web) { - String url = Utilities.pathURL(source, path)+"?nocache=" + System.currentTimeMillis(); - System.out.println("Fetch "+ url); - long t = System.currentTimeMillis(); - HTTPResult res = ManagedWebAccess.get(Arrays.asList("web"), url); - if (res.getCode() < 300 && res.getContent().length > 0) { - TextFile.bytesToFile(res.getContent(), df); - } - System.out.println(" ... done ("+stats(res.getContent().length, t)+")"); - } else { - File sf = new File(Utilities.path(source, path)); - if (sf.exists()) { - Utilities.copyFile(sf, df); - } - } - } - public void finish(String existingFilesBase, List existingFiles) throws IOException { - if (web) { - StringBuilder b = new StringBuilder(); - for (String s : existingFiles) { - b.append(existingFilesBase == null ? s : Utilities.path(existingFilesBase, s)); - b.append("\r\n"); - } - TextFile.stringToFile(b.toString(), deleteFileName()); - // for now, it must be done manually - if (upload) { - List filesToUpload = Utilities.listAllFiles(destination, null); - Collections.sort(filesToUpload, new UploadSorter()); // more specific files first - System.out.println("Ready to upload changes. "+filesToUpload.size()+" files to upload, "+existingFiles.size()+" files to delete"); - int t = filesToUpload.size()+existingFiles.size(); - System.out.println("Connect to "+uploadServer); - FTPClient ftp = new FTPClient(uploadServer, uploadPath, uploadUser, uploadPword); - ftp.connect(); - int lineLength = 0; - int count = 0; - long start = System.currentTimeMillis(); - if (!existingFiles.isEmpty()) { - System.out.print("Deleting"); - for (String s : existingFiles) { - count++; - lineLength = doProgressNote(s, count, start, existingFiles.size(), lineLength); - ftp.delete(existingFilesBase == null ? s : Utilities.path(existingFilesBase, s)); - } - } - System.out.println("Uploading"); - int failCount = 0; - count = 0; - start = System.currentTimeMillis(); - lineLength = 0; - for (String s : filesToUpload) { - count++; - lineLength = doProgressNote(s, count, start, filesToUpload.size(), lineLength); - - if (!s.contains(":")) { // hack around a bug that should be fixed elsewhere - try { - String fn = Utilities.path(destination, s); - ftp.upload(fn, s); - failCount = 0; - } catch (Exception e) { - System.out.println(""); - System.out.println("Error uploading file '"+s+"': "+e.getMessage()+". Trying again"); - try { - ftp.upload(Utilities.path(destination, s), s); - failCount = 0; - } catch (Exception e2) { - try { - System.out.println("Reconnecting after second error: "+e.getMessage()); - ftp = new FTPClient(uploadServer, uploadPath, uploadUser, uploadPword); - ftp.connect(); - ftp.upload(Utilities.path(destination, s), s); - } catch (Exception e3) { - failCount++; - System.out.println(""); - System.out.println("Error uploading file '"+s+"': "+e2.getMessage()); - System.out.println("Need to manually copy '"+Utilities.path(destination, s)+"' to '"+s); - if (failCount >= 10) { - throw new Error("Too many sequential errors copying files (10). Stopping."); - } - } - } - } - } - } - } - System.out.println(""); - System.out.println("Upload finished"); - } else { - System.out.println("Applying changes to website source at "+source); - for (String s : existingFiles) { - new File(Utilities.path(s, existingFilesBase, s)).delete(); - } - Utilities.copyDirectory(destination, source, null); - System.out.println(" ... done"); - } + File sf = new File(Utilities.path(source, path)); + if (sf.exists()) { + Utilities.copyFile(sf, df); + } } - private int doProgressNote(String file, int count, long start, int size, int length) throws IOException { - System.out.print(Utilities.padLeft("", '\b', length)); - String note; - - int pct = ((count * 50) / size); - String pc = "["+Utilities.padRight(Utilities.padLeft("", '#', pct), ' ', 50)+"]"; - long secondsDone = (System.currentTimeMillis() - start) / 1000; - if (count == 0 || secondsDone == 0) { - note = pc; - } else { - float rate = (count * 60) / secondsDone; - long secondsToGo = ((secondsDone * size) / count) - secondsDone; - Duration d = Duration.ofSeconds(secondsToGo); - note = pc+ "("+rate+" files/min, ~"+Utilities.describeDuration(d)+" to go) "+Utilities.getDirectoryForFile(file)+" "; + public void finish(String existingFilesBase, List existingFiles) throws IOException { + System.out.println("Applying changes to website source at "+source); + for (String s : existingFiles) { + new File(Utilities.path(s, existingFilesBase, s)).delete(); } - System.out.print(note); - System.out.flush(); - return note.length(); + Utilities.copyDirectory(destination, source, null); + System.out.println(" ... done"); } private String deleteFileName() throws IOException { @@ -296,27 +151,10 @@ private String deleteFileName() throws IOException { } public String instructions(int fc) throws IOException { - if (web) { - if (upload) { - return "The web site source at ftp://"+uploadServer+"/"+uploadPath+" has been updated"; - } else { - if (fc > 0) { - return "Upload all the files in "+destination+" to "+source+" using your preferred file copy method. Note that there is "+fc+" files to be deleted on the website (see "+deleteFileName()+")"; - - } else { - return "Upload all the files in "+destination+" to "+source+" using your preferred file copy method"; - } - } - } else { - return "The web site source in "+source+" has been updated"; - } - } - - public boolean isWeb() { - return web; + return "The web site source in "+source+" has been updated"; } public String verb() { - return web ? "Upload" : "Copy"; + return "Copy"; } }