OpenShift v4.x 환경에서 MongoDB Enterprise Operator를 구성하고 실제 배포하는 부분을 테스트 한다.

1. Project(Namespace) 생성

 

[root@bastion ~]# vi 00_namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
  name: mongodb
  annotations:
    openshift.io/node-selector: ""
  labels:
    openshift.io/cluster-monitoring: "false"
[root@bastion ~]# oc create -f 00_namespace.yaml

2. MongoDB Enterprise Operator 설치

 

[root@bastion ~]# vi 01_subscription.yaml
apiVersion: operators.coreos.com/v1alpha1
kind: Subscription
metadata:
  name: mongodb-enterprise
  namespace: mongodb
spec:
  channel: stable
  installPlanApproval: Automatic
  name: mongodb-enterprise
  source: certified-operators
  sourceNamespace: openshift-marketplace
[root@bastion ~]# oc create -f 01_subscription.yaml

3. SCC 권한 부여

 

privileged 권한을 부여한다.

[root@bastion ~]# for i in {default,mongodb-enterprise-ops-manager,mongodb-enterprise-appdb,mongodb-enterprise-database-pods}; do \
  oc adm policy add-scc-to-user privileged -z $i -n mongodb;
done

4. OpsManager Instance 배포

 

4.1. OpsManager Credentials 생성

 

[root@bastion ~]# vi 02_ops-manager-credentials.yaml
apiVersion: v1
kind: Secret
metadata:
  name: ops-manager-admin
  namespace: "mongodb"
type: Opaque
stringData:
  FirstName: KIM
  LastName: YONGBOK
  Password: test12#
  Username: your-id@example.com
[root@bastion ~]# oc create -f 02_ops-manager-credentials.yaml

4.2. OpsManager Instance 배포

 

[root@bastion ~]# vi 03_ops-manager.yaml
apiVersion: mongodb.com/v1
kind: MongoDBOpsManager
metadata:
  name: ops-manager
  namespace: mongodb
spec:
  version: 6.0.3
  replicas: 3
  adminCredentials: ops-manager-admin
  # 외부 서비스용 Port 노출 설정
  externalConnectivity:
    type: NodePort
    # Pod가 구동된 노드에서만 NodePort가 할당되게 한다.
    externalTrafficPolicy: Local
    port: 30000
  configuration:
    # OpsManager를 구성 후 접근시 UI 상에서 나머지 설정하는 부분을 무시.
    mms.ignoreInitialUiSetup: "true"
    # OpsManager의 관리자 메일 주소
    # mms.adminEmailAddr 항목은 반드시 필요.
    mms.adminEmailAddr: your-id@example.com
    mms.fromEmailAddr: your-id@example.com
    mms.replyToEmailAddr: your-id@example.com
    mms.mail.hostname: smtp-mail.outlook.com
    mms.mail.port: "587"
    mms.mail.ssl: "true"
    mms.mail.transport: smtp
    mms.minimumTLSVersion: TLSv1.2
  # OpsManager의 Resource Limit 설정
  statefulSet:
    spec:
      template:
        spec:
          containers:
            - name: mongodb-ops-manager
              resources:
                requests:
                  cpu: '8'
                  memory: 32Gi
                limits:
                  cpu: '8'
                  memory: 32Gi
          tolerations:
            - key: "key"
              operator: "Exists"
              effect: "NoSchedule"
  # OpsManager에서 사용되는 AppDB에 대한 설정
  applicationDatabase:
    # Standalone, ReplicaSet, ShardedCluster
    type: ReplicaSet
    members: 3
	# 로그 레벨: INFO, DEBUG, ERROR, WARN, FATAL
    logLevel: DEBUG
	# AppDB의 버전 설정
    version: 4.4.0-ent
	# AppDB 구동시 사용되는 Container Resource Limit 및 이미지 경로, PV 용량 설정
    podSpec:
      cpu: '8'
      memory: 32G
      podTemplate:
        spec:
          initContainers:
            - name: mongodb-enterprise-init-appdb
              image: 'quay.io/mongodb/mongodb-enterprise-init-appdb-ubi:1.0.14'
          containers:
            - name: mongod
              image: 'quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.4.0-ent'
            - name: mongodb-agent
              image: 'quay.io/mongodb/mongodb-agent-ubi:12.0.15.7646-1'
            - name: mongodb-agent-monitoring
              image: 'quay.io/mongodb/mongodb-agent-ubi:12.0.15.7646-1'
      persistence:
        multiple:
          data:
            storage: 4Ti
            storageClass: nfs-sc
          journal:
            storage: 4Ti
            storageClass: nfs-sc
          logs:
            storage: 4Ti
            storageClass: nfs-sc
    # Agent 설정
    agent:
      startupOptions:
        serverSelectionTimeoutSeconds: '20'
      logLevel: INFO
  # 백업 설정
  backup:
    headDB:
      storage: 4Ti
      storageClass: nfs-sc
    externalServiceEnabled: true
    members: 3
    enabled: false
[root@bastion ~]# oc create -f 03_ops-manager.yaml

4.3. OpsManager Route 생성

 

externalTrafficPolicy이 Local로 NodePort를 활성화 했기 때문에,
Pod가 구동된 노드에서만 접근이 가능하나, 보안상 권장하지 않으므로, Route를 생성하여 접근하도록 한다.

[root@bastion ~]# vi 04_ops-manager-route.yaml
kind: Route
apiVersion: route.openshift.io/v1
metadata:
  name: ops-manager
  namespace: mongodb
spec:
  host: ops-manager-mongodb.apps.ocp4.local
  to:
    kind: Service
    name: ops-manager-svc-ext
    weight: 100
  port:
    targetPort: 8080
  wildcardPolicy: None
[root@bastion ~]# oc create -f 04_ops-manager-route.yaml

4.4. OpsManager Pod 확인

 

  • OpsManager AppDB
[root@bastion ~]# oc get pod -l app=ops-manager-db-svc -n mongodb
NAME               READY   STATUS    RESTARTS   AGE
ops-manager-db-0   3/3     Running   0          115m
ops-manager-db-1   3/3     Running   0          115m
ops-manager-db-2   3/3     Running   0          116m
  • OpsManager
[root@bastion ~]# oc get pod -l app=ops-manager-svc -n mongodb
NAME            READY   STATUS    RESTARTS   AGE
ops-manager-0   1/1     Running   0          128m
ops-manager-1   1/1     Running   0          123m
ops-manager-2   1/1     Running   0          120m

4.5. OpsManager 설정

 

4.5.1. OpsManager 로그인 페이지

 

4.1. OpsManager Credentials 생성‘에서 생성한 정보를 기준으로 로그인 한다.
MongoDB - OpsManager: Login

4.5.2. Organizations 생성

 

상단 우측 -> FirstName(KIM) -> Organizations 선택.
MongoDB - OpsManager: Create Organizations #1

Create an Organization 선택.
MongoDB - OpsManager: Create Organizations #2

‘Name Your Organization’과 ‘Select a Default Server Type’을 선택 후 ‘Next’ 선택.
MongoDB - OpsManager: Create Organizations #3

‘Create Organization’ 선택.
MongoDB - OpsManager: Create Organizations #4

4.5.3. Project 생성

 

‘New Projet’ 선택.
MongoDB - OpsManager: Create Project #1

‘Name Your Organization’과 ‘Select a Default Server Type’을 선택 후 ‘Next’ 선택.
MongoDB - OpsManager: Create Project #2

‘Create Projet’ 선택.
MongoDB - OpsManager: Create Project #3

4.5.4. MongoDB – OpsManager ConfigMap 생성

 

MongoDB가 사용하는 OpsManager의 Organization ID와 서비스 URL을 확인 후 ConfigMap을 생성한다.

  • Organization ID 확인
    OpsManager의 상단 Organization 이름(openshift)을 선택 후 톱니 바퀴를 누른다.
    MongoDB - OpsManager: Organizations ID #1
    MongoDB - OpsManager: Organizations ID #2

  • OpsManager 내부 서비스 도메인 확인

[root@bastion ~]# oc get svc -n mongodb
NAME                  TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)           AGE
ops-manager-svc       ClusterIP   None             <none>        8080/TCP          67m
  • OpsManager ConfigMap 생성
[root@bastion ~]# vi 05_mongodb-ops-cloud-manager-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  # 히스토리 관리를 위해 OpsManager의 Project 이름과 같게 한다.
  name: ybkim-mongodb
  namespace: mongodb
data:
  projectName: ybkim-mongodb
  orgId: 64063b38f0c1811cc374b69a
  baseUrl: http://ops-manager-svc.mongodb.svc.cluster.local:8080
[root@bastion ~]# oc create -f 05_mongodb-ops-cloud-manager-configmap.yaml

4.5.5. Mongodb – Credentials Secret 생성

 

MongoDB가 OpsManager의 Project API Key 정보를 확인 후 생성한다.

  • OpsManager: Project API Key 생성
    상단 ‘Access Manager’를 선택 후 ‘Project Access(ybkim-mongodb)’를 선택.
    MongoDB - OpsManager: Create Project API Key #1

‘API Keys’ 탭 메뉴 -> ‘Create API Key’ 선택.
MongoDB - OpsManager: Create Project API Key #2

‘Description’란에 API Key에 대한 이름을 입력 후 ‘Project permissions’을 적절하게 선택 하고 ‘Next’ 선택.
MongoDB - OpsManager: Create Project API Key #3

‘Public Key’와 ‘Private Key’는 최초 한번만 보여주므로 따로 메모 해둔다.
이후 ‘Add Access List Entry’를 선택 한다.
MongoDB - OpsManager: Create Project API Key #4

‘Add Access List Entry’에 ClusterIP를 추가 한다.

[root@bastion ~]# oc get network -o yaml
apiVersion: v1
items:
- apiVersion: config.openshift.io/v1
  kind: Network
  metadata:
    name: cluster
  spec:
    clusterNetwork:
    - cidr: 111.111.0.0/16
      hostPrefix: 20
    externalIP:
      policy: {}
    networkType: OVNKubernetes
    serviceNetwork:
    - 100.100.0.0/16

MongoDB - OpsManager: Create Project API Key #5

최종적으로 ‘Done’을 선택하여 API Key 생성을 마무리 한다.
MongoDB - OpsManager: Create Project API Key #6

  • Credentials Secret 생성
    위 내용 확인 후 ‘user’는 ‘Public Key’ 내용을 적고, ‘publicApiKey’는 ‘Private Key’를 적으면 된다.
[root@bastion ~]# vi 06_mongodb-credentials.yaml
apiVersion: v1
kind: Secret
metadata:
  name: ybkim-mongodb-credentials
  namespace: "mongodb"
type: Opaque
stringData:
  user: dxgoilqc
  publicApiKey: 6f10f539-0569-4938-bef0-12dcb00f0753
[root@bastion ~]# oc create -f 06_mongodb-credentials.yaml

5. MongoDB Instance 생성

 

[root@bastion ~]# vi 07_mongodb-instance.yaml
apiVersion: mongodb.com/v1
kind: MongoDB
metadata:
  name: ybkim-mongodb-replica-set
spec:
  version: 4.4.0-ent
  type: ReplicaSet
  members: 3
  logLevel: INFO
  persistent: true
  exposedExternally: true
  opsManager:
    configMapRef:
      name: ybkim-mongodb
  credentials: ybkim-mongodb-credentials
  shardCount: 3
  configServerCount: 3
  mongosCount: 3
  mongodsPerShardCount: 3
  agent:
    startupOptions:
      maxLogFiles: "30"
      dialTimeoutSeconds: "40"
    logLevel: INFO
  statefulSet:
    spec:
      serviceName: "ybkim-mongodb"
  podSpec:
    podTemplate:
      spec:
        initContainers:
        - name: mongodb-enterprise-init-database
          image: 'quay.io/mongodb/mongodb-enterprise-init-database-ubi:1.0.14'
        containers:
          - name: mongodb-enterprise-database
            image: 'quay.io/mongodb/mongodb-enterprise-database-ubi:2.0.2'
            resources:
              limits:
                cpu: "4"
                memory: 16Gi
    persistence:
      multiple:
        data:
          storage: 2Ti
          storageClass: nfs-sc
        journal:
          storage: 2Ti
          storageClass: nfs-sc
        logs:
          storage: 2Ti
          storageClass: nfs-sc
  mongosPodSpec:
    persistence:
      multiple:
        data:
          storageClass: nfs-sc
          storage: 2Ti
        journal:
          storageClass: nfs-sc
          storage: 2Ti
        logs:
          storageClass: nfs-sc
          storage: 2Ti
  shardPodSpec:
    persistence:
      multiple:
        data:
          storage: 2Ti
          storageClass: nfs-sc
        journal:
          storage: 2Ti
          storageClass: nfs-sc
        logs:
          storage: 2Ti
          storageClass: nfs-sc
  configSrvPodSpec:
    persistence:
      multiple:
        data:
          storage: 2Ti
          storageClass: nfs-sc
        journal:
          storage: 2Ti
          storageClass: nfs-sc
        logs:
          storage: 2Ti
          storageClass: nfs-sc
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchExpressions:
          - key: node-role.kubernetes.io/worker
            operator: In
            values:
            - ''
  backup:
    autoTerminateOnDeletion: true
    mode: disabled
[root@bastion ~]# oc create -f 07_mongodb-instance.yaml

5.1. MongoDB Pod 확인

[root@bastion ~]# oc get pod -l app=ybkim-mongodb -n mongodb
NAME                          READY   STATUS    RESTARTS   AGE
ybkim-mongodb-replica-set-0   1/1     Running   0          12m
ybkim-mongodb-replica-set-1   1/1     Running   0          13m
ybkim-mongodb-replica-set-2   1/1     Running   0          12m

5.2. OpsManager – Project 확인
MongoDB - OpsManager: Project Deployment Process
MongoDB - OpsManager: Project Deployment Server
MongoDB - OpsManager: Project Deployment Overview
MongoDB - OpsManager: Project Deployment Real Time
MongoDB - OpsManager: Project Deployment Metrics

끝.