import * as core from '@actions/core' import * as KubernetesConstants from '../types/kubernetesTypes.js' import {Kubectl, Resource} from '../types/kubectl.js' import {checkForErrors} from './kubectlUtils.js' import {sleep} from './timeUtils.js' import {ResourceTypeFleet} from '../actions/deploy.js' import {ClusterType} from '../inputUtils.js' const IS_SILENT = false const POD = 'pod' export async function checkManifestStability( kubectl: Kubectl, resources: Resource[], resourceType: ClusterType, timeout?: string ): Promise { // Skip if resource type is microsoft.containerservice/fleets if (resourceType === ResourceTypeFleet) { core.info(`Skipping checkManifestStability for ${ResourceTypeFleet}`) return } let rolloutStatusHasErrors = false // Collect errors for reporting // This will be used to throw a detailed error at the end if any rollout fails // This is useful for debugging and understanding which resources failed // their rollout status check // It will also include the describe output for the resource that failed // to provide more context on the failure const rolloutErrors: string[] = [] for (let i = 0; i < resources.length; i++) { const resource = resources[i] if ( KubernetesConstants.WORKLOAD_TYPES_WITH_ROLLOUT_STATUS.indexOf( resource.type.toLowerCase() ) >= 0 ) { try { const result = await kubectl.checkRolloutStatus( resource.type, resource.name, resource.namespace, timeout ) checkForErrors([result]) } catch (ex) { const errorMessage = `Rollout failed for ${resource.type}/${resource.name} in namespace ${kubectl.getNamespace(resource.namespace)}: ${ex.message || ex}` core.error(errorMessage) rolloutErrors.push(errorMessage) // Get more detailed information try { const describeResult = await kubectl.describe( resource.type, resource.name, IS_SILENT, resource.namespace ) core.info( `Describe output for ${resource.type}/${resource.name}:\n${describeResult.stdout}` ) } catch (describeEx) { core.warning( `Could not describe ${resource.type}/${resource.name}: ${describeEx}` ) } rolloutStatusHasErrors = true } } if ( resource.type.toLowerCase() === KubernetesConstants.KubernetesWorkload.POD.toLowerCase() ) { try { await exports.checkPodStatus(kubectl, resource) } catch (ex) { core.warning( `Could not determine pod status: ${JSON.stringify(ex)}` ) await kubectl.describe( resource.type, resource.name, IS_SILENT, resource.namespace ) } } if ( resource.type == KubernetesConstants.DiscoveryAndLoadBalancerResource.SERVICE ) { try { const service = await getService(kubectl, resource) const {spec, status} = service if (spec.type === KubernetesConstants.ServiceTypes.LOAD_BALANCER) { if (!isLoadBalancerIPAssigned(status)) { await waitForServiceExternalIPAssignment(kubectl, resource) } else { core.info( `ServiceExternalIP ${resource.name} ${status.loadBalancer.ingress[0].ip}` ) } } } catch (ex) { const errorMessage = `Could not determine service status of: ${resource.name} in namespace ${kubectl.getNamespace(resource.namespace)}. Error: ${ex.message || ex}` core.warning(errorMessage) try { const describeResult = await kubectl.describe( resource.type, resource.name, IS_SILENT, resource.namespace ) core.info( `Describe output for service/${resource.name}:\n${describeResult.stdout}` ) } catch (describeEx) { core.warning( `Could not describe service/${resource.name}: ${describeEx}` ) } } } } if (rolloutStatusHasErrors) { const detailedError = `Rollout status failed for the following resources:\n${rolloutErrors.join('\n')}` throw new Error(detailedError) } } export async function checkPodStatus( kubectl: Kubectl, pod: Resource ): Promise { const sleepTimeout = 10 * 1000 // 10 seconds const iterations = 60 // 60 * 10 seconds timeout = 10 minutes max timeout let podStatus let kubectlDescribeNeeded = false let errorDetails = '' for (let i = 0; i < iterations; i++) { await sleep(sleepTimeout) core.debug(`Polling for pod status: ${pod.name}`) podStatus = await getPodStatus(kubectl, pod) if ( podStatus && podStatus?.phase !== 'Pending' && podStatus?.phase !== 'Unknown' ) { break } } podStatus = await getPodStatus(kubectl, pod) // Get container statuses for detailed error information const containerErrors = getContainerErrors(podStatus) switch (podStatus.phase) { case 'Succeeded': case 'Running': if (isPodReady(podStatus)) { console.log(`pod/${pod.name} is successfully rolled out`) } else { errorDetails = `Pod ${pod.name} is ${podStatus.phase} but not ready. ${containerErrors}` core.error(errorDetails) kubectlDescribeNeeded = true } break case 'Pending': if (!isPodReady(podStatus)) { errorDetails = `Pod ${pod.name} rollout status check timed out (still Pending after ${(iterations * sleepTimeout) / 1000} seconds). ${containerErrors}` core.warning(errorDetails) kubectlDescribeNeeded = true } break case 'Failed': errorDetails = `Pod ${pod.name} rollout failed. ${containerErrors}` core.error(errorDetails) kubectlDescribeNeeded = true break default: errorDetails = `Pod ${pod.name} has unexpected status: ${podStatus.phase}. ${containerErrors}` core.warning(errorDetails) kubectlDescribeNeeded = true } if (kubectlDescribeNeeded) { try { const describeResult = await kubectl.describe( POD, pod.name, IS_SILENT, pod.namespace ) core.info( `Describe output for pod/${pod.name}:\n${describeResult.stdout}` ) } catch (describeEx) { core.warning(`Could not describe pod/${pod.name}: ${describeEx}`) } // Throw error with detailed information if (errorDetails) { throw new Error(errorDetails) } } } async function getPodStatus(kubectl: Kubectl, pod: Resource) { const podResult = await kubectl.getResource( POD, pod.name, IS_SILENT, pod.namespace ) checkForErrors([podResult]) return JSON.parse(podResult.stdout).status } function isPodReady(podStatus: any): boolean { let allContainersAreReady = true podStatus.containerStatuses.forEach((container) => { if (container.ready === false) { core.info( `'${container.name}' status: ${JSON.stringify(container.state)}` ) allContainersAreReady = false } }) if (!allContainersAreReady) { core.warning('All containers not in ready state') } return allContainersAreReady } export function getContainerErrors(podStatus: any): string { const errors: string[] = [] const collectErrors = (containers: any[], label: string) => { containers?.forEach(({name, ready, state}) => { if (ready) return if (state?.waiting) { errors.push( `${label} '${name}' is waiting: ${state.waiting.reason} - ${state.waiting.message || 'No message'}` ) } else if (state?.terminated) { errors.push( `${label} '${name}' terminated: ${state.terminated.reason} - ${state.terminated.message || 'No message'}` ) } else { errors.push( `${label} '${name}' is not ready: ${JSON.stringify(state)}` ) } }) } collectErrors(podStatus.containerStatuses, 'Container') collectErrors(podStatus.initContainerStatuses, 'Init container') return errors.length ? `Container issues: ${errors.join('; ')}` : '' } async function getService(kubectl: Kubectl, service: Resource) { const serviceResult = await kubectl.getResource( KubernetesConstants.DiscoveryAndLoadBalancerResource.SERVICE, service.name, IS_SILENT, service.namespace ) checkForErrors([serviceResult]) return JSON.parse(serviceResult.stdout) } async function waitForServiceExternalIPAssignment( kubectl: Kubectl, service: Resource ): Promise { const sleepTimeout = 10 * 1000 // 10 seconds const iterations = 18 // 18 * 10 seconds timeout = 3 minutes max timeout for (let i = 0; i < iterations; i++) { core.info(`Wait for service ip assignment : ${service.name}`) await sleep(sleepTimeout) const status = (await getService(kubectl, service)).status if (isLoadBalancerIPAssigned(status)) { core.info( `ServiceExternalIP ${service.name} ${status.loadBalancer.ingress[0].ip}` ) return } } core.warning(`Wait for service ip assignment timed out ${service.name}`) } function isLoadBalancerIPAssigned(status: any) { return status?.loadBalancer?.ingress?.length > 0 }