OpenShift v4.x 환경에서 MongoDB Enterprise Operator를 구성하고 실제 배포하는 부분을 테스트 한다.
1. Project(Namespace) 생성
[root@bastion ~]# vi 00_namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
name: mongodb
annotations:
openshift.io/node-selector: ""
labels:
openshift.io/cluster-monitoring: "false"
[root@bastion ~]# oc create -f 00_namespace.yaml
2. MongoDB Enterprise Operator 설치
[root@bastion ~]# vi 01_subscription.yaml
apiVersion: operators.coreos.com/v1alpha1
kind: Subscription
metadata:
name: mongodb-enterprise
namespace: mongodb
spec:
channel: stable
installPlanApproval: Automatic
name: mongodb-enterprise
source: certified-operators
sourceNamespace: openshift-marketplace
[root@bastion ~]# oc create -f 01_subscription.yaml
3. SCC 권한 부여
privileged 권한을 부여한다.
[root@bastion ~]# for i in {default,mongodb-enterprise-ops-manager,mongodb-enterprise-appdb,mongodb-enterprise-database-pods}; do \
oc adm policy add-scc-to-user privileged -z $i -n mongodb;
done
4. OpsManager Instance 배포
4.1. OpsManager Credentials 생성
[root@bastion ~]# vi 02_ops-manager-credentials.yaml
apiVersion: v1
kind: Secret
metadata:
name: ops-manager-admin
namespace: "mongodb"
type: Opaque
stringData:
FirstName: KIM
LastName: YONGBOK
Password: test12#
Username: your-id@example.com
[root@bastion ~]# oc create -f 02_ops-manager-credentials.yaml
4.2. OpsManager Instance 배포
[root@bastion ~]# vi 03_ops-manager.yaml
apiVersion: mongodb.com/v1
kind: MongoDBOpsManager
metadata:
name: ops-manager
namespace: mongodb
spec:
version: 6.0.3
replicas: 3
adminCredentials: ops-manager-admin
# 외부 서비스용 Port 노출 설정
externalConnectivity:
type: NodePort
# Pod가 구동된 노드에서만 NodePort가 할당되게 한다.
externalTrafficPolicy: Local
port: 30000
configuration:
# OpsManager를 구성 후 접근시 UI 상에서 나머지 설정하는 부분을 무시.
mms.ignoreInitialUiSetup: "true"
# OpsManager의 관리자 메일 주소
# mms.adminEmailAddr 항목은 반드시 필요.
mms.adminEmailAddr: your-id@example.com
mms.fromEmailAddr: your-id@example.com
mms.replyToEmailAddr: your-id@example.com
mms.mail.hostname: smtp-mail.outlook.com
mms.mail.port: "587"
mms.mail.ssl: "true"
mms.mail.transport: smtp
mms.minimumTLSVersion: TLSv1.2
# OpsManager의 Resource Limit 설정
statefulSet:
spec:
template:
spec:
containers:
- name: mongodb-ops-manager
resources:
requests:
cpu: '8'
memory: 32Gi
limits:
cpu: '8'
memory: 32Gi
tolerations:
- key: "key"
operator: "Exists"
effect: "NoSchedule"
# OpsManager에서 사용되는 AppDB에 대한 설정
applicationDatabase:
# Standalone, ReplicaSet, ShardedCluster
type: ReplicaSet
members: 3
# 로그 레벨: INFO, DEBUG, ERROR, WARN, FATAL
logLevel: DEBUG
# AppDB의 버전 설정
version: 4.4.0-ent
# AppDB 구동시 사용되는 Container Resource Limit 및 이미지 경로, PV 용량 설정
podSpec:
cpu: '8'
memory: 32G
podTemplate:
spec:
initContainers:
- name: mongodb-enterprise-init-appdb
image: 'quay.io/mongodb/mongodb-enterprise-init-appdb-ubi:1.0.14'
containers:
- name: mongod
image: 'quay.io/mongodb/mongodb-enterprise-appdb-database-ubi:4.4.0-ent'
- name: mongodb-agent
image: 'quay.io/mongodb/mongodb-agent-ubi:12.0.15.7646-1'
- name: mongodb-agent-monitoring
image: 'quay.io/mongodb/mongodb-agent-ubi:12.0.15.7646-1'
persistence:
multiple:
data:
storage: 4Ti
storageClass: nfs-sc
journal:
storage: 4Ti
storageClass: nfs-sc
logs:
storage: 4Ti
storageClass: nfs-sc
# Agent 설정
agent:
startupOptions:
serverSelectionTimeoutSeconds: '20'
logLevel: INFO
# 백업 설정
backup:
headDB:
storage: 4Ti
storageClass: nfs-sc
externalServiceEnabled: true
members: 3
enabled: false
[root@bastion ~]# oc create -f 03_ops-manager.yaml
4.3. OpsManager Route 생성
externalTrafficPolicy이 Local로 NodePort를 활성화 했기 때문에,
Pod가 구동된 노드에서만 접근이 가능하나, 보안상 권장하지 않으므로, Route를 생성하여 접근하도록 한다.
[root@bastion ~]# vi 04_ops-manager-route.yaml
kind: Route
apiVersion: route.openshift.io/v1
metadata:
name: ops-manager
namespace: mongodb
spec:
host: ops-manager-mongodb.apps.ocp4.local
to:
kind: Service
name: ops-manager-svc-ext
weight: 100
port:
targetPort: 8080
wildcardPolicy: None
[root@bastion ~]# oc create -f 04_ops-manager-route.yaml
4.4. OpsManager Pod 확인
- OpsManager AppDB
[root@bastion ~]# oc get pod -l app=ops-manager-db-svc -n mongodb
NAME READY STATUS RESTARTS AGE
ops-manager-db-0 3/3 Running 0 115m
ops-manager-db-1 3/3 Running 0 115m
ops-manager-db-2 3/3 Running 0 116m
- OpsManager
[root@bastion ~]# oc get pod -l app=ops-manager-svc -n mongodb
NAME READY STATUS RESTARTS AGE
ops-manager-0 1/1 Running 0 128m
ops-manager-1 1/1 Running 0 123m
ops-manager-2 1/1 Running 0 120m
4.5. OpsManager 설정
4.5.1. OpsManager 로그인 페이지
‘4.1. OpsManager Credentials 생성‘에서 생성한 정보를 기준으로 로그인 한다.
4.5.2. Organizations 생성
상단 우측 -> FirstName(KIM) -> Organizations 선택.
Create an Organization 선택.
‘Name Your Organization’과 ‘Select a Default Server Type’을 선택 후 ‘Next’ 선택.
‘Create Organization’ 선택.
4.5.3. Project 생성
‘New Projet’ 선택.
‘Name Your Organization’과 ‘Select a Default Server Type’을 선택 후 ‘Next’ 선택.
‘Create Projet’ 선택.
4.5.4. MongoDB – OpsManager ConfigMap 생성
MongoDB가 사용하는 OpsManager의 Organization ID와 서비스 URL을 확인 후 ConfigMap을 생성한다.
-
Organization ID 확인
OpsManager의 상단 Organization 이름(openshift)을 선택 후 톱니 바퀴를 누른다.
-
OpsManager 내부 서비스 도메인 확인
[root@bastion ~]# oc get svc -n mongodb
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
ops-manager-svc ClusterIP None <none> 8080/TCP 67m
- OpsManager ConfigMap 생성
[root@bastion ~]# vi 05_mongodb-ops-cloud-manager-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
# 히스토리 관리를 위해 OpsManager의 Project 이름과 같게 한다.
name: ybkim-mongodb
namespace: mongodb
data:
projectName: ybkim-mongodb
orgId: 64063b38f0c1811cc374b69a
baseUrl: http://ops-manager-svc.mongodb.svc.cluster.local:8080
[root@bastion ~]# oc create -f 05_mongodb-ops-cloud-manager-configmap.yaml
4.5.5. Mongodb – Credentials Secret 생성
MongoDB가 OpsManager의 Project API Key 정보를 확인 후 생성한다.
- OpsManager: Project API Key 생성
상단 ‘Access Manager’를 선택 후 ‘Project Access(ybkim-mongodb)’를 선택.
‘API Keys’ 탭 메뉴 -> ‘Create API Key’ 선택.
‘Description’란에 API Key에 대한 이름을 입력 후 ‘Project permissions’을 적절하게 선택 하고 ‘Next’ 선택.
‘Public Key’와 ‘Private Key’는 최초 한번만 보여주므로 따로 메모 해둔다.
이후 ‘Add Access List Entry’를 선택 한다.
‘Add Access List Entry’에 ClusterIP를 추가 한다.
[root@bastion ~]# oc get network -o yaml
apiVersion: v1
items:
- apiVersion: config.openshift.io/v1
kind: Network
metadata:
name: cluster
spec:
clusterNetwork:
- cidr: 111.111.0.0/16
hostPrefix: 20
externalIP:
policy: {}
networkType: OVNKubernetes
serviceNetwork:
- 100.100.0.0/16
최종적으로 ‘Done’을 선택하여 API Key 생성을 마무리 한다.
- Credentials Secret 생성
위 내용 확인 후 ‘user’는 ‘Public Key’ 내용을 적고, ‘publicApiKey’는 ‘Private Key’를 적으면 된다.
[root@bastion ~]# vi 06_mongodb-credentials.yaml
apiVersion: v1
kind: Secret
metadata:
name: ybkim-mongodb-credentials
namespace: "mongodb"
type: Opaque
stringData:
user: dxgoilqc
publicApiKey: 6f10f539-0569-4938-bef0-12dcb00f0753
[root@bastion ~]# oc create -f 06_mongodb-credentials.yaml
5. MongoDB Instance 생성
[root@bastion ~]# vi 07_mongodb-instance.yaml
apiVersion: mongodb.com/v1
kind: MongoDB
metadata:
name: ybkim-mongodb-replica-set
spec:
version: 4.4.0-ent
type: ReplicaSet
members: 3
logLevel: INFO
persistent: true
exposedExternally: true
opsManager:
configMapRef:
name: ybkim-mongodb
credentials: ybkim-mongodb-credentials
shardCount: 3
configServerCount: 3
mongosCount: 3
mongodsPerShardCount: 3
agent:
startupOptions:
maxLogFiles: "30"
dialTimeoutSeconds: "40"
logLevel: INFO
statefulSet:
spec:
serviceName: "ybkim-mongodb"
podSpec:
podTemplate:
spec:
initContainers:
- name: mongodb-enterprise-init-database
image: 'quay.io/mongodb/mongodb-enterprise-init-database-ubi:1.0.14'
containers:
- name: mongodb-enterprise-database
image: 'quay.io/mongodb/mongodb-enterprise-database-ubi:2.0.2'
resources:
limits:
cpu: "4"
memory: 16Gi
persistence:
multiple:
data:
storage: 2Ti
storageClass: nfs-sc
journal:
storage: 2Ti
storageClass: nfs-sc
logs:
storage: 2Ti
storageClass: nfs-sc
mongosPodSpec:
persistence:
multiple:
data:
storageClass: nfs-sc
storage: 2Ti
journal:
storageClass: nfs-sc
storage: 2Ti
logs:
storageClass: nfs-sc
storage: 2Ti
shardPodSpec:
persistence:
multiple:
data:
storage: 2Ti
storageClass: nfs-sc
journal:
storage: 2Ti
storageClass: nfs-sc
logs:
storage: 2Ti
storageClass: nfs-sc
configSrvPodSpec:
persistence:
multiple:
data:
storage: 2Ti
storageClass: nfs-sc
journal:
storage: 2Ti
storageClass: nfs-sc
logs:
storage: 2Ti
storageClass: nfs-sc
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/worker
operator: In
values:
- ''
backup:
autoTerminateOnDeletion: true
mode: disabled
[root@bastion ~]# oc create -f 07_mongodb-instance.yaml
5.1. MongoDB Pod 확인
[root@bastion ~]# oc get pod -l app=ybkim-mongodb -n mongodb
NAME READY STATUS RESTARTS AGE
ybkim-mongodb-replica-set-0 1/1 Running 0 12m
ybkim-mongodb-replica-set-1 1/1 Running 0 13m
ybkim-mongodb-replica-set-2 1/1 Running 0 12m
5.2. OpsManager – Project 확인
끝.