[OCTRL-1091] Integrate k8s client into ECS core

common/controlmode/controlmode.go

@@ -41,6 +41,8 @@ const (
 	HOOK
 	KUBECTL_DIRECT
 	KUBECTL_FAIRMQ
+	KUBERNETES_DIRECT
+	KUBERNETES_FAIRMQ
 )
 
 func (cm ControlMode) String() string {
@@ -57,6 +59,10 @@ func (cm ControlMode) String() string {
 		return "kubectl_direct"
 	case KUBECTL_FAIRMQ:
 		return "kubectl_fairmq"
+	case KUBERNETES_DIRECT:
+		return "kubernetes_direct"
+	case KUBERNETES_FAIRMQ:
+		return "kubernetes_fairmq"
 	}
 	return "direct"
 }
@@ -81,6 +87,10 @@ func (cm *ControlMode) UnmarshalText(b []byte) error {
 		*cm = KUBECTL_DIRECT
 	case "kubectl_fairmq":
 		*cm = KUBECTL_FAIRMQ
+	case "kubernetes_direct":
+		*cm = KUBERNETES_DIRECT
+	case "kubernetes_fairmq":
+		*cm = KUBERNETES_FAIRMQ
 	default:
 		*cm = DIRECT
 	}

control-operator/api/v1alpha1/environment_types.go

@@ -82,7 +82,7 @@ type TaskReference struct {
 	Name           string            `json:"name"`
 	Env            []v1.EnvVar       `json:"env"`
 	ArgsCLI        []string          `json:"argsCLI"`
-	ArgsTransition map[string]string `json:"argsTransition"`
+	ArgsTransition map[string]string `json:"argsTransition,omitempty"`
 }
 
 type TemplateSpecification struct {

control-operator/config/crd/bases/aliecs.alice.cern_environments.yaml

@@ -4102,7 +4102,6 @@ spec:
                         type: string
                     required:
                     - argsCLI
-                    - argsTransition
                     - env
                     - name
                     type: object

control-operator/ecs-manifests/control-workflows/readout-docker.yaml

@@ -7,7 +7,7 @@ defaults:
   _module_cmdline: >-
     source /etc/profile.d/modules.sh && MODULEPATH={{ modulepath }} module load Readout Control-OCCPlugin &&
     o2-readout-exe
-  _plain_cmdline: "sudo -E docker run --name readout --replace --privileged --user flp -v /etc/group:/etc/group:ro -v /etc/passwd:/etc/passwd:ro -v /tmp:/tmp -v /lib/modules/$(uname -r):/lib/modules/$(uname -r) --network=host --ipc=host -e O2_DETECTOR -e O2_PARTITION -e OCC_CONTROL_PORT -e O2_SYSTEM -e O2_ROLE gitlab-registry.cern.ch/aliceo2group/dockerfiles/alma9-flp-node:2 /opt/o2/bin/o2-readout-exe"
+  _plain_cmdline: "sudo -E docker run --name readout --replace --privileged --user flp -v /etc/group:/etc/group:ro -v /etc/passwd:/etc/passwd:ro -v /tmp:/tmp -v /lib/modules/$(uname -r):/lib/modules/$(uname -r) --network=host --ipc=host -e O2_DETECTOR -e O2_PARTITION -e OCC_CONTROL_PORT -e O2_SYSTEM -e O2_ROLE gitlab-registry.cern.ch/aliceo2group/dockerfiles/alma9-flp-node/readout:1 /opt/o2/bin/o2-readout-exe"
 control:
   mode: direct
 wants:

control-operator/ecs-manifests/control-workflows/readout-kube-direct.yaml

@@ -0,0 +1,33 @@
+name: readout # read by workflow
+defaults:
+  readout_cfg_uri: "consul-ini://{{ consul_endpoint }}/o2/components/readout/ANY/any/readout-standalone-{{ task_hostname }}"
+  user: flp
+  log_task_stdout: none
+  log_task_stderr: none
+  _module_cmdline: >-
+    source /etc/profile.d/modules.sh && MODULEPATH={{ modulepath }} module load Readout Control-OCCPlugin &&
+    o2-readout-exe
+  _plain_cmdline: "{{ o2_install_path }}/bin/o2-readout-exe"
+control:
+  mode: kubernetes_direct
+wants:
+  cpu: 0.15
+  memory: 128
+bind:
+  - name: readout
+    type: push
+    rateLogging: "{{ fmq_rate_logging }}"
+    addressing: ipc
+    transport: shmem
+properties: {}
+command: 
+  stdout: "{{ log_task_stdout }}"
+  stderr: "{{ log_task_stderr }}"
+  shell: true
+  env:
+    - O2_DETECTOR={{ detector }}
+    - O2_PARTITION={{ environment_id }}
+  user: "{{ user }}"
+  arguments:
+    - "{{ readout_cfg_uri }}"
+  value: "{{ len(modulepath)>0 ? _module_cmdline : _plain_cmdline }}"

control-operator/ecs-manifests/control-workflows/stfbuilder-senderoutput-kube-direct.yaml

@@ -0,0 +1,64 @@
+name: stfbuilder-senderoutput
+control:
+  mode: kubernetes_fairmq
+defaults:
+  detector: TST
+  rdh_version: "6"
+  user: flp
+  fmq_severity: info
+  dd_detector_subspec: feeid
+  dd_discovery_endpoint: "no-op://" # location of consul docker instance
+  log_task_stdout: none
+  log_task_stderr: none
+  stfb_dd_mode: "physics"
+  stfb_fee_mask: "0xffff"
+  stfb_enable_datasink: "false"
+  stfb_datasink_dir: "/tmp"
+  stfb_datasink_filesize: "2048"
+  monitoring_dd_interval: "5"
+  _module_cmdline: >-
+    source /etc/profile.d/modules.sh && MODULEPATH={{ modulepath }} module load DataDistribution Control-OCCPlugin &&
+    StfBuilder
+  _plain_cmdline: "{{ o2_install_path }}/bin/StfBuilder"
+wants:
+  cpu: 0.15
+  memory: 128
+bind:
+  - name: buildertosender
+    type: push
+    rateLogging: "{{ fmq_rate_logging }}"
+    transport: shmem
+    addressing: ipc
+    sndBufSize: "4"
+command:
+  env:
+    - O2_INFOLOGGER_MODE={{infologger_mode}}
+    - O2_DETECTOR={{ detector }}
+    - DATADIST_FEE_MASK={{ stfb_fee_mask }}
+    - O2_PARTITION={{ environment_id }}
+  stdout: "{{ log_task_stdout }}"
+  stderr: "{{ log_task_stderr }}"
+  shell: true
+  user: "{{ user }}"
+  value: "{{ len(modulepath)>0 ? _module_cmdline : _plain_cmdline }}"
+  arguments:
+    - "--session=default"
+    - "--transport=shmem"
+    - "--shm-segment-id=2"
+    - "--shm-segment-size=33554432"
+    - "--monitoring-backend={{ monitoring_dd_url }}"
+    - "--monitoring-process-interval={{ monitoring_dd_interval }}"
+    - "--discovery-partition={{ environment_id }}"
+    - "--discovery-endpoint={{ dd_discovery_endpoint }}"
+    - "--detector-rdh={{ rdh_version }}"
+    - "--detector-subspec={{ dd_detector_subspec }}"
+    - "--severity={{ fmq_severity }}"
+    - "--severity-infologger={{ infologger_severity }}"
+    - "--output-channel-name=buildertosender"
+    - "--shm-monitor=false"
+    - "--detector={{ detector }}"
+    - "--run-type={{ stfb_dd_mode }}"
+    - "{{ stfb_enable_datasink == 'true' ? '--data-sink-enable' : ' ' }}"
+    - "{{ stfb_enable_datasink == 'true' ? '--data-sink-dir=' +  stfb_datasink_dir : ' ' }}"
+    - "{{ stfb_enable_datasink == 'true' ? '--data-sink-max-stfs-per-file=0' : ' ' }}"
+    - "{{ stfb_enable_datasink == 'true' ? '--data-sink-max-file-size=' +  stfb_datasink_filesize : ' ' }}"

control-operator/ecs-manifests/control-workflows/stfsender-kube-direct.yaml

@@ -0,0 +1,68 @@
+name: stfsender
+control:
+  mode: kubernetes_fairmq
+wants:
+  cpu: 0.15
+  memory: 128
+defaults:
+  max_buffered_stfs: "-1"
+  dd_discovery_net_if: "ib0" #the loopback intf
+  dd_discovery_endpoint: "no-op://" # location of consul docker instance
+  user: flp
+  fmq_severity: info
+  log_task_stdout: none
+  log_task_stderr: none
+  stfs_dd_region_size: 4096
+  stfs_shm_segment_size: 33554432
+  stfs_enable_datasink: "false"
+  stfs_datasink_dir: "/tmp"
+  stfs_datasink_filesize: "2048"
+  monitoring_dd_interval: "5"
+  _module_cmdline: >-
+    source /etc/profile.d/modules.sh && MODULEPATH={{ modulepath }} module load DataDistribution Control-OCCPlugin &&
+    numactl --cpunodebind=0 --preferred=0 -- StfSender
+  _plain_cmdline: "numactl"
+command:
+  env:
+    - O2_INFOLOGGER_MODE={{infologger_mode}}
+    - http_proxy=""
+    - https_proxy=""
+    - no_proxy=-ib,.internal
+    - O2_DETECTOR={{ detector }}
+    - UCX_NET_DEVICES=mlx5_0:1 # This limits StfSender to IB interface (used as of DD v1.3.0)
+    - UCX_TLS=sm,self,dc,rc    # Force dc/rc connection (used as of DD v1.4.0)
+    - UCX_IB_SL=1              # Force IB SL1 with Adaptive Routing (AR)
+    - UCX_DC_MAX_NUM_EPS=512   # Allow 512 peers for DC transport
+    - UCX_RC_MAX_NUM_EPS=512   # Allow 512 peers for RC transport
+    - O2_PARTITION={{ environment_id }}
+  stdout: "{{ log_task_stdout }}"
+  stderr: "{{ log_task_stderr }}"
+  shell: true
+  user: "{{ user }}"
+  value: "{{ len(modulepath)>0 ? _module_cmdline : _plain_cmdline }}"
+  arguments:
+    - "--cpunodebind=0"
+    - "--preferred=0"
+    - "--" 
+    - "{{ o2_install_path }}/bin/StfSender"
+    - "--session=default"
+    - "--shm-segment-id=2"
+    - "--shm-segment-size={{ stfs_shm_segment_size }}"
+    - "--dd-region-size={{ stfs_dd_region_size }}"
+    - "--dd-region-id=3536"
+    - "--transport=shmem"
+    - "--input-channel-name={{ stfs_input_channel_name }}"
+    - "--severity={{ fmq_severity }}"
+    - "--severity-infologger={{ infologger_severity }}"
+    - "--monitoring-backend={{ monitoring_dd_url }}"
+    - "--monitoring-process-interval={{ monitoring_dd_interval }}"
+    - "--discovery-partition={{ environment_id }}"
+    - "--discovery-id={{ dd_discovery_stfs_id }}"
+    - "--discovery-endpoint={{ dd_discovery_endpoint }}"
+    - "--discovery-net-if={{ dd_discovery_net_if }}"
+    - '{{ ddsched_enabled == "true" ? "" : "--stand-alone" }}'
+    - "--shm-monitor=false"
+    - "{{ stfs_enable_datasink == 'true' ? '--data-sink-enable' : ' ' }}"
+    - "{{ stfs_enable_datasink == 'true' ? '--data-sink-dir=' +  stfs_datasink_dir : ' ' }}"
+    - "{{ stfs_enable_datasink == 'true' ? '--data-sink-max-stfs-per-file=0' : ' ' }}"
+    - "{{ stfs_enable_datasink == 'true' ? '--data-sink-max-file-size=' +  stfs_datasink_filesize : ' ' }}"