Narsing Shah

📌 OCI OKE Infra Automation

8/5/2020

Image Hardening

  1. The images we use are usually hardened to meet certain security & audit requirements
  2. Hence these images are provided by our OS Management team
  3. This script helps such teams to use the Hardened Image

Setup Steps

  1. OCI CLI & Authentication: The OCI CLI must be installed and configured (oci setup config) on the machine running this script.
  2. Oracle Cloud Agent (For NFS Mount Automation): The Compute Instance Run Command plugin must be enabled on both the old and new compute hosts. This allows the script to query and run mount commands without needing SSH keys or direct network access.
  3. Required IAM Policies: The executing user must have permissions to manage custom-images, instances, instance-agent-commands, volume-attachments, and kubernetes-clusters in the target compartment.

Bash Script (infra_automation.sh)

#!/usr/bin/env bash
#
# OCI Infrastructure Automation Script
#
# This script automates:
#   1. Importing a custom image from a PAR URL
#   2. Launching a compute instance using the imported image
#   3. Querying NFS mount info from an old VM and mounting it on the new VM
#   4. Shutting down the old VM
#   5. Creating an OKE Node Pool using the imported image
#   6. Deleting the OKE Node Pool after successful creation
#

set -euo pipefail

# -----------------------------------------------------------------------------
# Configuration / Variables (Modify with your specific OCIDs)
# -----------------------------------------------------------------------------
COMPARTMENT_OCID="ocid1.compartment.oc1.phx.aaaaaaaasamplecompartment12345"
AD_NAME="Uocm:PHX-AD-1"
SUBNET_OCID="ocid1.subnet.oc1.phx.aaaaaaaasamplesubnet12345"
SHAPE="VM.Standard.E4.Flex"
SHAPE_CONFIG='{"ocpus": 1, "memoryInGBs": 16}'  # Config for Flex shape
SSH_PUBLIC_KEY_FILE="$HOME/.ssh/id_rsa.pub"      # Path to your public key

# Image Import Settings
PAR_URL="https://objectstorage.us-phoenix-1.oraclecloud.com/p/sample-par-token/n/namespace/b/bucket/o/custom-image.qcow2"
IMAGE_DISPLAY_NAME="custom-imported-image-$(date +%s)"

# Compute Hosts
OLD_VM_OCID="ocid1.instance.oc1.phx.aaaaaaaasampleoldvm12345"
INSTANCE_NAME="new-compute-host-$(date +%s)"

# OKE Node Pool Settings
CLUSTER_OCID="ocid1.cluster.oc1.phx.aaaaaaaasamplecluster12345"
NODE_POOL_NAME="oke-node-pool-$(date +%s)"
# Placement details required for node launching
PLACEMENT_CONFIGS='[{"availabilityDomain": "'"$AD_NAME"'", "subnetId": "'"$SUBNET_OCID"'"}]'


# -----------------------------------------------------------------------------
# 1. Fetch & Import Custom Image from PAR URL
# -----------------------------------------------------------------------------
echo "========================================================================"
echo "Step 1: Importing custom image from Object Storage PAR URL..."
echo "========================================================================"

# The CLI directly fetches the image from the PAR URL and starts the import
oci compute image import from-object-uri \
  --compartment-id "$COMPARTMENT_OCID" \
  --uri "$PAR_URL" \
  --display-name "$IMAGE_DISPLAY_NAME" \
  --launch-mode PARAVIRTUALIZED \
  --source-image-type QCOW2 \
  --wait-for-state AVAILABLE \
  --max-wait-seconds 1800 # 30 min timeout for image import

echo "Custom image imported successfully as AVAILABLE."


# -----------------------------------------------------------------------------
# 2. Get Imported Image OCID and Start New Compute Host
# -----------------------------------------------------------------------------
echo "========================================================================"
echo "Step 2: Retrieving image OCID and launching new compute host..."
echo "========================================================================"

IMAGE_OCID=$(oci compute image list \
  --compartment-id "$COMPARTMENT_OCID" \
  --display-name "$IMAGE_DISPLAY_NAME" \
  --query "data[0].id" \
  --raw-output)

echo "Discovered Image OCID: $IMAGE_OCID"

echo "Launching new compute host: $INSTANCE_NAME..."
NEW_INSTANCE_OCID=$(oci compute instance launch \
  --compartment-id "$COMPARTMENT_OCID" \
  --availability-domain "$AD_NAME" \
  --shape "$SHAPE" \
  --shape-config "$SHAPE_CONFIG" \
  --image-id "$IMAGE_OCID" \
  --subnet-id "$SUBNET_OCID" \
  --display-name "$INSTANCE_NAME" \
  --ssh-authorized-keys-file "$SSH_PUBLIC_KEY_FILE" \
  --wait-for-state RUNNING \
  --query "data.id" \
  --raw-output)

echo "New compute host launched successfully. OCID: $NEW_INSTANCE_OCID"


# -----------------------------------------------------------------------------
# 3. Mount /commonstorage NFS Mountpoint from Old VM to New VM
# -----------------------------------------------------------------------------
echo "========================================================================"
echo "Step 3: Transferring and mounting NFS /commonstorage mountpoint..."
echo "========================================================================"

echo "Running query command on the old VM to fetch the NFS mount source..."
QUERY_CMD_ID=$(oci compute instance-agent-command create-instance-agent-command \
  --compartment-id "$COMPARTMENT_OCID" \
  --instance-id "$OLD_VM_OCID" \
  --execution-type "RUN_SCRIPT" \
  --source '{"sourceType": "MESSAGE_CONTENT", "commandContent": "grep \"/commonstorage\" /etc/fstab || findmnt -n -o SOURCE,FSTYPE,OPTIONS /commonstorage"}' \
  --query "data.id" \
  --raw-output)

# Poll until query command completes
while true; do
  STATUS=$(oci compute instance-agent-command get-instance-agent-command-execution \
    --instance-agent-command-id "$QUERY_CMD_ID" \
    --instance-id "$OLD_VM_OCID" \
    --query "data.lifecycleState" \
    --raw-output)
  if [ "$STATUS" = "SUCCEEDED" ]; then
    break
  elif [ "$STATUS" = "FAILED" ] || [ "$STATUS" = "CANCELED" ]; then
    echo "ERROR: Could not query NFS mount information from the old VM."
    exit 1
  fi
  sleep 5
done

# Read standard output containing the fstab mount details
NFS_MOUNT_LINE=$(oci compute instance-agent-command get-instance-agent-command-execution \
  --instance-agent-command-id "$QUERY_CMD_ID" \
  --instance-id "$OLD_VM_OCID" \
  --query "data.content.message" \
  --raw-output)

# Extract NFS export server and path (e.g. 10.0.0.10:/commonstorage_export) and mount options
NFS_SOURCE=$(echo "$NFS_MOUNT_LINE" | awk '{print $1}')
NFS_OPTIONS=$(echo "$NFS_MOUNT_LINE" | awk '{print $4}')

# Fallback in case old VM does not return valid output
if [ -z "$NFS_SOURCE" ]; then
  echo "WARNING: Could not auto-detect NFS mount on old VM. Using placeholder values."
  NFS_SOURCE="10.0.0.10:/commonstorage_export"
  NFS_OPTIONS="nfsvers=3,noacl,defaults,_netdev"
fi

echo "Detected NFS Source: $NFS_SOURCE"
echo "Detected Mount Options: $NFS_OPTIONS"

# Build mounting script for the new VM
MOUNT_SCRIPT="sudo mkdir -p /commonstorage && sudo mount -t nfs -o $NFS_OPTIONS $NFS_SOURCE /commonstorage && echo \"$NFS_SOURCE /commonstorage nfs $NFS_OPTIONS 0 0\" | sudo tee -a /etc/fstab"

echo "Executing mount command on the new VM..."
MOUNT_CMD_ID=$(oci compute instance-agent-command create-instance-agent-command \
  --compartment-id "$COMPARTMENT_OCID" \
  --instance-id "$NEW_INSTANCE_OCID" \
  --execution-type "RUN_SCRIPT" \
  --source "{\"sourceType\": \"MESSAGE_CONTENT\", \"commandContent\": \"$MOUNT_SCRIPT\"}" \
  --query "data.id" \
  --raw-output)

# Poll until mount command completes on the new instance
while true; do
  STATUS=$(oci compute instance-agent-command get-instance-agent-command-execution \
    --instance-agent-command-id "$MOUNT_CMD_ID" \
    --instance-id "$NEW_INSTANCE_OCID" \
    --query "data.lifecycleState" \
    --raw-output)
  if [ "$STATUS" = "SUCCEEDED" ]; then
    echo "NFS mount attached and configured persistently in fstab."
    break
  elif [ "$STATUS" = "FAILED" ] || [ "$STATUS" = "CANCELED" ]; then
    echo "ERROR: Failed to mount NFS storage on the new VM."
    exit 1
  fi
  sleep 5
done


# -----------------------------------------------------------------------------
# 4. Shutdown the Old VM
# -----------------------------------------------------------------------------
echo "========================================================================"
echo "Step 4: Gracefully shutting down the old VM..."
echo "========================================================================"

oci compute instance action \
  --instance-id "$OLD_VM_OCID" \
  --action SOFTSTOP \
  --wait-for-state STOPPED

echo "Old VM has been stopped successfully."


# -----------------------------------------------------------------------------
# 5. Create a New OKE Node Pool
# -----------------------------------------------------------------------------
echo "========================================================================"
echo "Step 5: Creating OKE Node Pool using image OCID..."
echo "========================================================================"

echo "Starting node pool creation: $NODE_POOL_NAME..."
oci ce node-pool create \
  --compartment-id "$COMPARTMENT_OCID" \
  --cluster-id "$CLUSTER_OCID" \
  --name "$NODE_POOL_NAME" \
  --node-shape "$SHAPE" \
  --node-shape-config "$SHAPE_CONFIG" \
  --node-source-details '{"imageId": "'"$IMAGE_OCID"'", "sourceType": "IMAGE"}' \
  --placement-configs "$PLACEMENT_CONFIGS" \
  --size 2

echo "Waiting for Node Pool registration..."
sleep 15

# Retrieve the newly created node pool OCID by listing node pools in the cluster
NODE_POOL_OCID=$(oci ce node-pool list \
  --compartment-id "$COMPARTMENT_OCID" \
  --cluster-id "$CLUSTER_OCID" \
  --query "data[?name=='$NODE_POOL_NAME'].id | [0]" \
  --raw-output)

echo "Discovered Node Pool OCID: $NODE_POOL_OCID"

# Poll until OKE Node Pool state reaches ACTIVE
while true; do
  STATUS=$(oci ce node-pool get \
    --node-pool-id "$NODE_POOL_OCID" \
    --query "data.\"lifecycle-state\"" \
    --raw-output)
  echo "Current Node Pool lifecycle state: $STATUS"
  if [ "$STATUS" = "ACTIVE" ]; then
    echo "Node pool is now ACTIVE and ready."
    break
  elif [ "$STATUS" = "FAILED" ]; then
    echo "ERROR: Node pool creation failed."
    exit 1
  fi
  sleep 20
done


# -----------------------------------------------------------------------------
# 6. Delete OKE Node Pool
# -----------------------------------------------------------------------------
echo "========================================================================"
echo "Step 6: Deleting OKE Node Pool..."
echo "========================================================================"

oci ce node-pool delete \
  --node-pool-id "$NODE_POOL_OCID" \
  --force

# Poll until the node pool is deleted
while true; do
  # If a resource is deleted, oci get will return 404 or state DELETED
  STATUS=$(oci ce node-pool get \
    --node-pool-id "$NODE_POOL_OCID" \
    --query "data.\"lifecycle-state\"" 2>/dev/null || echo "DELETED")
  echo "Current Node Pool lifecycle state: $STATUS"
  if [ "$STATUS" = "DELETED" ]; then
    break
  fi
  sleep 20
done

echo "========================================================================"
echo "Execution completed successfully!"
echo "========================================================================"