@DataBoundConstructor public NomadSlaveTemplate( String cpu, String memory, String disk, String labels, String remoteFs, String idleTerminationInMinutes, String region, String priority, String image ) { this.cpu = Integer.parseInt(cpu); this.memory = Integer.parseInt(memory); this.disk = Integer.parseInt(disk); this.priority = Integer.parseInt(priority); this.idleTerminationInMinutes = Integer.parseInt(idleTerminationInMinutes); this.remoteFs = remoteFs; this.labels = Util.fixNull(labels); this.labelSet = Label.parse(labels); this.region = region; this.image = image; readResolve(); }
@Override public Collection<NodeProvisioner.PlannedNode> provision(Label label, int excessWorkload) { List<NodeProvisioner.PlannedNode> nodes = new ArrayList<>(); final NomadSlaveTemplate template = getTemplate(label); try { while (excessWorkload > 0) { LOGGER.log(Level.INFO, "Excess workload, provisioning new Jenkins slave on Nomad cluster"); final String slaveName = template.createSlaveName(); nodes.add(new NodeProvisioner.PlannedNode( slaveName, NomadComputer.threadPoolForRemoting.submit( new ProvisioningCallback(slaveName, template, this) ), template.getNumExecutors())); excessWorkload -= template.getNumExecutors(); } return nodes; } catch (Exception e) { LOGGER.log(Level.SEVERE, "Unable to schedule new Jenkins slave on Nomad cluster, message: " + e.getMessage()); } return Collections.emptyList(); }
@Override @Restricted(DoNotUse.class) public void onStarted(Cloud cloud, Label label, Collection<NodeProvisioner.PlannedNode> plannedNodes) { BulkChange change = new BulkChange(stats); try { boolean changed = false; for (NodeProvisioner.PlannedNode plannedNode : plannedNodes) { ProvisioningActivity.Id id = getIdFor(plannedNode); if (id != null) { onStarted(id); changed = true; } } if (changed) { // Not using change.commit() here as persist handles exceptions already stats.persist(); } } finally { change.abort(); } }
@Override public Collection<NodeProvisioner.PlannedNode> provision(Label label, int excessWorkload) { try { List<NodeProvisioner.PlannedNode> r = new ArrayList<NodeProvisioner.PlannedNode>(); final ECSTaskTemplate template = getTemplate(label); for (int i = 1; i <= excessWorkload; i++) { r.add(new NodeProvisioner.PlannedNode(template.getDisplayName(), Computer.threadPoolForRemoting .submit(new ProvisioningCallback(template, label)), 1)); } return r; } catch (Exception e) { LOGGER.log(Level.WARNING, "Failed to provision ECS slave", e); return Collections.emptyList(); } }
@Nonnull public List<DockerCloud> getDockerClouds(Label label) { List<DockerCloud> provisionClouds; //create a random list of docker clouds and prioritize idle clouds List<DockerCloud> availableClouds = getAvailableDockerClouds(label); if (availableClouds.size() > 0) { //select available clouds based on label which have potential capacity LOG.debug("Picking from available clouds."); provisionClouds = availableClouds; } else { //if there's no available clouds then fall back to original behavior LOG.debug("Falling back to getting all clouds regardless of availability."); provisionClouds = getAllDockerClouds(); } //randomize the order of the DockerCloud list Collections.shuffle(provisionClouds); //sort by least loaded DockerCloud (i.e. fewest provisioned slaves) provisionClouds.sort(new DockerCloudLoadComparator()); LOG.debug("Least loaded randomized DockerCloud: " + ((provisionClouds.size() > 0) ? provisionClouds.get(0).name : "none available")); return provisionClouds; }
/** * Multiple templates may have the same label. * * @return Templates matched to requested label assuming slave Mode */ @Nonnull public List<DockerSlaveTemplate> getTemplates(Label label) { List<DockerSlaveTemplate> dockerSlaveTemplates = new ArrayList<>(); for (DockerSlaveTemplate t : templates) { if (isNull(label) && t.getMode() == Node.Mode.NORMAL) { dockerSlaveTemplates.add(t); } if (nonNull(label) && label.matches(t.getLabelSet())) { dockerSlaveTemplates.add(t); } } return dockerSlaveTemplates; }
/** * Initializes data structure that we don't persist. */ @SuppressWarnings("unused") public Object readResolve() { if (configVersion < 1) { if (isNull(nodeProperties)) nodeProperties = new ArrayList<>(); nodeProperties.add(new DockerNodeProperty("DOCKER_CONTAINER_ID", "JENKINS_CLOUD_ID", "DOCKER_HOST")); configVersion = 1; } // real @Nonnull if (mode == null) { mode = Node.Mode.NORMAL; } if (retentionStrategy == null) { retentionStrategy = new DockerOnceRetentionStrategy(10); } try { labelSet = Label.parse(getLabelString()); // fails sometimes under debugger } catch (Throwable t) { LOG.error("Can't parse labels: {}", t); } return this; }
/** * Initializes data structure that we don't persist. */ protected Object readResolve() { labelSet = Label.parse(labels); securityGroupSet = parseSecurityGroups(); /** * In releases of this plugin prior to 1.18, template-specific instance caps could be configured * but were not enforced. As a result, it was possible to have the instance cap for a template * be configured to 0 (zero) with no ill effects. Starting with version 1.18, template-specific * instance caps are enforced, so if a configuration has a cap of zero for a template, no instances * will be launched from that template. Since there is no practical value of intentionally setting * the cap to zero, this block will override such a setting to a value that means 'no cap'. */ if (instanceCap == 0) { instanceCap = Integer.MAX_VALUE; } if (amiType == null) { amiType = new UnixData(rootCommandPrefix, sshPort); } return this; }
@DataBoundConstructor public RavelloSlaveBlueprint(String blueprintName, int blueprintId, String slaveName, String remoteFS, String nodeDescription, RetentionStrategy retentionStrategy, int port, String credentialsId, String jvmOptions, String javaPath, String prefixStartSlaveCmd, String suffixStartSlaveCmd, Integer launchTimeoutSeconds, Integer maxNumRetries, Integer retryWaitTime, String labelString){ super(blueprintName, blueprintId); this.slaveName = slaveName; this.remoteFS = remoteFS; this.nodeDescription = nodeDescription; this.retentionStrategy = retentionStrategy; this.port = port; this.credentialsId = credentialsId; this.jvmOptions = jvmOptions; this.javaPath = javaPath; this.prefixStartSlaveCmd = prefixStartSlaveCmd; this.suffixStartSlaveCmd = suffixStartSlaveCmd; this.launchTimeoutSeconds = launchTimeoutSeconds; this.maxNumRetries = maxNumRetries; this.retryWaitTime = retryWaitTime; this.labelString = labelString; labelSet = Label.parse(labelString); }
public SlaveTemplate getTemplate(Label label) { if (label == null && templates.size() > 0) { return templates.get(0); } if ("master".equals(label.getName())) { if (templates.size() > 0) { return templates.get(0); } else { return null; } } for (SlaveTemplate t : templates) { if(label == null || label.matches(t.getLabelSet())) { return t; } } return null; }
/** * Multiple amis can have the same label. * * @return Templates matched to requested label assuming slave Mode */ public List<DockerTemplate> getTemplates(Label label) { ArrayList<DockerTemplate> dockerTemplates = new ArrayList<>(); for (DockerTemplate t : templates) { if (label == null && t.getMode() == Node.Mode.NORMAL) { dockerTemplates.add(t); } if (label != null && label.matches(t.getLabelSet())) { dockerTemplates.add(t); } } // add temporary templates matched to requested label for (DockerTemplate template : getJobTemplates().values()) { if (label != null && label.matches(template.getLabelSet())) { dockerTemplates.add(template); } } return dockerTemplates; }
@DataBoundConstructor public DockerTemplate(@Nonnull DockerTemplateBase dockerTemplateBase, DockerComputerConnector connector, String labelString, String remoteFs, String instanceCapStr ) { this.dockerTemplateBase = dockerTemplateBase; this.connector = connector; this.labelString = Util.fixNull(labelString); this.remoteFs = Strings.isNullOrEmpty(remoteFs) ? "/home/jenkins" : remoteFs; if (instanceCapStr.equals("")) { this.instanceCap = Integer.MAX_VALUE; } else { this.instanceCap = Integer.parseInt(instanceCapStr); } labelSet = Label.parse(labelString); }
protected void should_connect_agent(DockerTemplate template) throws IOException, ExecutionException, InterruptedException { // FIXME on CI windows nodes don't have Docker4Windows Assume.assumeFalse(SystemUtils.IS_OS_WINDOWS); String dockerHost = SystemUtils.IS_OS_WINDOWS ? "tcp://localhost:2375" : "unix:///var/run/docker.sock"; DockerCloud cloud = new DockerCloud("docker", new DockerAPI(new DockerServerEndpoint(dockerHost, null)), Collections.singletonList(template)); j.jenkins.clouds.replaceBy(Collections.singleton(cloud)); final FreeStyleProject project = j.createFreeStyleProject("test-docker-ssh"); project.setAssignedLabel(Label.get("docker-agent")); project.getBuildersList().add(new Shell("whoami")); final FreeStyleBuild build = project.scheduleBuild2(0).get(); Assert.assertTrue(build.getResult() == Result.SUCCESS); Assert.assertTrue(build.getLog().contains("jenkins")); }
/** * Set the default Slave Info Label if no Label is assigned to the {@link Item} * @param item */ private void setLabel(final Item item) { if (item instanceof AbstractProject) { AbstractProject<?, ?> job = (AbstractProject<?, ?>) item; LOGGER.fine("MesosListener.setLabel(), setting label"); Label label = job.getAssignedLabel(); try { if (label == null) { // No label assigned, override now LOGGER.log(Level.FINE, "No label assigned to job - " + job.getDisplayName() + ". Assigning a label now..."); label = getLabel(); if (label != null) { LOGGER.log(Level.INFO, "Assigned \"" + label.getName() + "\" to job \"" + job.getDisplayName() + "\""); job.setAssignedLabel(label); } } } catch (Exception e) { LOGGER.log(Level.WARNING, "Failed to assign label \"" + label + "\" to " + job.getDisplayName(), e); } } }
/** * Get the default Slave Info Label as Hudson {@link hudson.model.Label} * @return */ private Label getLabel() { Label label = null; // get mesos cloud MesosCloud cloud = MesosCloud.get(); if(cloud != null) { // get all label associate with cloud List<MesosSlaveInfo> list = cloud.getSlaveInfos(); if(list != null && list.size() > 0) { for (MesosSlaveInfo slaveInfo: list) { if (slaveInfo.isDefaultSlave()) { label = Hudson.getInstance().getLabel(slaveInfo.getLabelString()); break; } } } } return label; }
@Override public boolean canProvision(Label label) { // Provisioning is simply creating a task for a jenkins slave. // We can provision a Mesos slave as long as the job's label matches any // item in the list of configured Mesos labels. // TODO(vinod): The framework may not have the resources necessary // to start a task when it comes time to launch the slave. if (slaveInfos != null) { for (MesosSlaveInfo slaveInfo : slaveInfos) { if (slaveInfo.matchesLabel(label)) { return true; } } } return false; }
/** * Verify the input label expression * * @param value * @return */ public FormValidation doCheckLabelExpression(@QueryParameter String value) { if(StringUtils.isBlank(value)) { return FormValidation.error(Messages.BuildPreference_labelExpression_requied()); } try { Label l = LabelExpression.parseExpression(value); if(l.getNodes().isEmpty()) { return FormValidation.warning(Messages.BuildPreference_labelExpression_empty()); } } catch(ANTLRException e) { return FormValidation.error(e, Messages.BuildPreference_labelExpression_invalid()); } return FormValidation.ok(); }
@Test public void testScriptedPipeline() throws Exception { String agentLabel = "my-agent"; jenkins.createOnlineSlave(Label.get(agentLabel)); WorkflowJob job = jenkins.createProject(WorkflowJob.class, "test-scripted-pipeline"); String pipelineScript = "node {\n" + " greet '" + name + "'\n" + "}"; job.setDefinition(new CpsFlowDefinition(pipelineScript, true)); WorkflowRun completedBuild = jenkins.assertBuildStatusSuccess(job.scheduleBuild2(0)); String expectedString = "Hello, " + name + "!"; jenkins.assertLogContains(expectedString, completedBuild); }
public NomadSlaveTemplate getTemplate(Label label) { for (NomadSlaveTemplate t : templates) { if (label == null && !t.getLabelSet().isEmpty()) { continue; } if ((label == null && t.getLabelSet().isEmpty()) || (label != null && label.matches(t.getLabelSet()))) { return t; } } return null; }
@BeforeClass public static void setUp() throws Exception { node1 = j.createSlave(Label.get("linux")); node2 = j.createSlave(Label.get("test")); Disk disk = new Disk(DISK_ID_ONE, null, "mount-from-master", PATH_ON_DISK, null); DiskPool diskPool = new DiskPool(DISK_POOL_ID, null, null, null, null, null, Collections.singletonList(disk)); setUpDiskPools(j.jenkins, diskPool); }
@Override public synchronized Collection<NodeProvisioner.PlannedNode> provision( final Label label, final int excessWorkload) { final FleetStateStats stats=updateStatus(); final int maxAllowed = this.getMaxSize(); if (stats.getNumDesired() >= maxAllowed || !"active".equals(stats.getState())) return Collections.emptyList(); int targetCapacity = stats.getNumDesired() + excessWorkload; if (targetCapacity > maxAllowed) targetCapacity = maxAllowed; int toProvision = targetCapacity - stats.getNumDesired(); LOGGER.log(Level.INFO, "Provisioning nodes. Excess workload: " + Integer.toString(excessWorkload) + ", Provisioning: " + Integer.toString(toProvision)); final ModifySpotFleetRequestRequest request=new ModifySpotFleetRequestRequest(); request.setSpotFleetRequestId(fleet); request.setTargetCapacity(targetCapacity); final AmazonEC2 ec2=connect(credentialsId, region); ec2.modifySpotFleetRequest(request); final List<NodeProvisioner.PlannedNode> resultList = new ArrayList<NodeProvisioner.PlannedNode>(); for(int f=0;f<toProvision; ++f) { final SettableFuture<Node> futureNode=SettableFuture.create(); final NodeProvisioner.PlannedNode plannedNode= new NodeProvisioner.PlannedNode("FleetNode-"+f, futureNode, 1); resultList.add(plannedNode); this.plannedNodes.add(plannedNode); } return resultList; }
@Override public Collection<NodeProvisioner.PlannedNode> provision(Label label, int excessWorkload) { assert provision != null; provision.j = j; int i = seq.getAndIncrement(); provision.id = new ProvisioningActivity.Id(name, null, name + "-slave-" + i); return Collections.<NodeProvisioner.PlannedNode>singletonList(new TrackedPlannedNode( provision.id, 1, Computer.threadPoolForRemoting.submit(provision) )); }
@Override public Collection<NodeProvisioner.PlannedNode> provision(Label label, int excessWorkload) { LOGGER.log(Level.SEVERE, "Going to provision " + excessWorkload + " executors"); Collection<NodeProvisioner.PlannedNode> result = new ArrayList<NodeProvisioner.PlannedNode>(); final ParallelsDesktopConnectorSlaveComputer connector = getConnector(); for (int i = 0; (i < vms.size()) && (excessWorkload > 0); i++) { final ParallelsDesktopVM vm = vms.get(i); if (vm.isProvisioned()) continue; if (!label.matches(Label.parse(vm.getLabels()))) continue; final String vmId = vm.getVmid(); final String slaveName = name + " " + vmId; vm.setSlaveName(slaveName); vm.setProvisioned(true); --excessWorkload; result.add(new NodeProvisioner.PlannedNode(slaveName, Computer.threadPoolForRemoting.submit(new Callable<Node>() { @Override public Node call() throws Exception { connector.checkVmExists(vmId); return connector.createSlaveOnVM(vm); } }), 1)); } return result; }
@Override public boolean canProvision(Label label) { if (label != null) { for (ParallelsDesktopVM vm : vms) { if (label.matches(Label.parse(vm.getLabels()))) return true; } } return false; }
private ECSTaskTemplate getTemplate(Label label) { for (ECSTaskTemplate t : templates) { if (label == null || label.matches(t.getLabelSet())) { return t; } } return null; }
@Override public boolean canProvision(Label label) { // Check if we have an image that matches the label. final DockerImage foundImage = findDockerImageForLabel(label); if (foundImage == null) { LOGGER.log(Level.FINE, "No matching labels."); return false; } else { return true; } }
private DockerImage findDockerImageForLabel(Label label) { for (DockerImage image : getImages()) { if (dockerImageMatchesLabel(image, label)) { return image; } } return null; }
/** * Used to validate the configurations in regards to the specified * {@link Label} and the {@link Node}s associated with it. * * @param build * the current build * @throws Exception * @since 1.0 */ private static void checkNodes(AbstractBuild<?, ?> build) throws Exception { Label label = ((DTBuild) build).getTestJob().getAssignedLabel(); // Check if the Nodes label is set if (label == null) { throw PrintMessages.throwLabelIsNotSet(); } // Check if the Job is being build on the Master if (build.getBuiltOn().toComputer() instanceof MasterComputer) { throw PrintMessages.throwCannotBuildOnMaster(); } }
@Nonnull @Override public List<DockerCloud> getDockerClouds(Label label) { return getInstance().clouds.stream() .filter(Objects::nonNull) .filter(DockerCloud.class::isInstance) .map(cloud -> (DockerCloud) cloud) .collect(Collectors.toList()); }
/** * Get a list of available DockerCloud clouds which are not at max * capacity. * * @param label A label expression of a Job Run requiring an executor. * @return A list of available DockerCloud clouds. */ protected List<DockerCloud> getAvailableDockerClouds(Label label) { return getAllDockerClouds().stream() .filter(cloud -> cloud.canProvision(label) && (countCurrentDockerSlaves(cloud) >= 0) && (countCurrentDockerSlaves(cloud) < cloud.getContainerCap())) .collect(Collectors.toList()); }
/** * Gets first {@link DockerSlaveTemplate} that has the matching {@link Label}. */ @CheckForNull public DockerSlaveTemplate getTemplate(Label label) { List<DockerSlaveTemplate> labelTemplates = getTemplates(label); if (!labelTemplates.isEmpty()) { return labelTemplates.get(0); } return null; }
/** * Gets {@link SlaveTemplate} that has the matching {@link Label}. */ public SlaveTemplate getTemplate(Label label) { for (SlaveTemplate t : templates) { if (t.getMode() == Node.Mode.NORMAL) { if (label == null || label.matches(t.getLabelSet())) { return t; } } else if (t.getMode() == Node.Mode.EXCLUSIVE) { if (label != null && label.matches(t.getLabelSet())) { return t; } } } return null; }
public void testLabelExpression() throws Exception{ setUpCloud(LABEL1 + " " + LABEL2); assertEquals(true, ac.canProvision(Label.parseExpression(LABEL1 + " || " + LABEL2))); assertEquals(true, ac.canProvision(Label.parseExpression(LABEL1 + " && " + LABEL2))); assertEquals(true, ac.canProvision(Label.parseExpression(LABEL1 + " || aaa"))); assertEquals(false, ac.canProvision(Label.parseExpression(LABEL1 + " && aaa"))); assertEquals(false, ac.canProvision(Label.parseExpression("aaa || bbb"))); assertEquals(false, ac.canProvision(Label.parseExpression("aaa || bbb"))); }
/** * Gets the {@link PodTemplate} by {@link Label}. * @param label The label. * @param templates The list of all templates. * @return The first pod template from the collection that has a matching label. */ public static PodTemplate getTemplateByLabel(@CheckForNull Label label, Collection<PodTemplate> templates) { for (PodTemplate t : templates) { if ((label == null && t.getNodeUsageMode() == Node.Mode.NORMAL) || (label != null && label.matches(t.getLabelSet()))) { return t; } } return null; }
/** * Check not too many already running. * */ private boolean addProvisionedSlave(@Nonnull PodTemplate template, @CheckForNull Label label) throws Exception { if (containerCap == 0) { return true; } KubernetesClient client = connect(); String templateNamespace = template.getNamespace(); // If template's namespace is not defined, take the // Kubernetes Namespace. if (Strings.isNullOrEmpty(templateNamespace)) { templateNamespace = client.getNamespace(); } PodList slaveList = client.pods().inNamespace(templateNamespace).withLabels(getLabels()).list(); List<Pod> slaveListItems = slaveList.getItems(); Map<String, String> labelsMap = getLabelsMap(template.getLabelSet()); PodList namedList = client.pods().inNamespace(templateNamespace).withLabels(labelsMap).list(); List<Pod> namedListItems = namedList.getItems(); if (slaveListItems != null && containerCap <= slaveListItems.size()) { LOGGER.log(Level.INFO, "Total container cap of {0} reached, not provisioning: {1} running or errored in namespace {2} with Kubernetes labels {3}", new Object[] { containerCap, slaveListItems.size(), client.getNamespace(), getLabels() }); return false; } if (namedListItems != null && slaveListItems != null && template.getInstanceCap() <= namedListItems.size()) { LOGGER.log(Level.INFO, "Template instance cap of {0} reached for template {1}, not provisioning: {2} running or errored in namespace {3} with label \"{4}\" and Kubernetes labels {5}", new Object[] { template.getInstanceCap(), template.getName(), slaveListItems.size(), client.getNamespace(), label == null ? "" : label.toString(), labelsMap }); return false; // maxed out } return true; }
/** * Gets all PodTemplates that have the matching {@link Label}. * @param label label to look for in templates * @return list of matching templates */ public ArrayList<PodTemplate> getMatchingTemplates(@CheckForNull Label label) { ArrayList<PodTemplate> podList = new ArrayList<PodTemplate>(); List<PodTemplate> podTemplates = getAllTemplates(); for (PodTemplate t : podTemplates) { if ((label == null && t.getNodeUsageMode() == Node.Mode.NORMAL) || (label != null && label.matches(t.getLabelSet()))) { podList.add(t); } } return podList; }
public Set<LabelAtom> getLabels() { if (_labels == null) { // Do not do this in readResolve as it can result in a recursive dependency load that // makes jenkins startup slow and unstable. _labels = Label.parse(this.labelString); } return _labels; }
public FormValidation doCheckLabelString(@QueryParameter String value) { value = nullToEmpty(value).trim(); if (value.length() == 0) { return FormValidation.error("Required"); } try { Label.parse(value); return FormValidation.ok(); } catch (Throwable ex) { return FormValidation.error("Invalid labels: %s", ex.getMessage()); } }
protected Object readResolve() { _nextHostsRefresh = new Instant(0); _jenkins = Jenkins.getInstance(); _hosts = newHashMap(); _credentialsProvider = new SshCredentialsProvider(_jenkins, _credentialsId); _labels = unmodifiableSet(Label.parse(_labelString)); _requiredLabels = unmodifiableSet(Label.parse(_requiredLabelString)); _directoryMappings = parseDirectoryMappings(_directoryMappingString); return this; }
public FormValidation doCheckRequiredLabelString(@QueryParameter String value) { if (isNullOrEmpty(value)) { return FormValidation.ok(); } try { Label.parse(value); return FormValidation.ok(); } catch (Throwable ex) { return FormValidation.error(ex.getMessage()); } }