Kafka CDC Connect

소프트웨어학과 202037634 윤주은

카카오엔터프라이즈 CDC 파일럿 설치 실패

Import library를 했을 때, 더 이상 지원되지 않는 github 주소가 많아 설치 실패하였음.
추후 개인 github에 go 패키지를 업로드하여 트러블 슈팅을 시도해볼 예정

1. Docker 컨테이너 구성

docker-compose.yml

version: '3'
services:
  mysql:
    image: mysql:8.0
    container_name: mysql
    ports:
      - 3306:3306
    environment:
      MYSQL_ROOT_PASSWORD: admin
      MYSQL_USER: mysqluser
      MYSQL_PASSWORD: mysqlpw
    command:
      - --character-set-server=utf8mb4
      - --collation-server=utf8mb4_unicode_ci
    volumes:
      - .docker/mysql/data:/var/lib/mysql

  zookeeper:
    container_name: zookeeper
    image: wurstmeister/zookeeper
    ports:
      - "2181:2181"

	kafka:
    container_name: kafka
    image: wurstmeister/kafka
    depends_on:
      - zookeeper
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
      KAFKA_ADVERTISED_HOST_NAME: 172.16.210.89
      KAFKA_ADVERTISED_PORT: 9092
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock

  mysql-sink:
    image: mysql:8.0
    container_name: mysql-sink
    ports:
      - 3307:3306
    environment:
      MYSQL_ROOT_PASSWORD: admin
      MYSQL_USER: mysqluser
      MYSQL_PASSWORD: mysqlpw
		command:
      - --character-set-server=utf8mb4
      - --collation-server=utf8mb4_unicode_ci
    volumes:
          - .docker/mysql-sink/data:/var/lib/mysql

docker-compose

docker-compose up -d
docker-compose -f {docker-compose 파일 이름} up -d

Untitled

2. MySQL 사전 설정

docker-compose.yml에서 구축한 mysql 컨테이너를 실행해서 진행한다.

# mysql 컨테이너 실행
docker exec -it mysql bash

# mysql에 로그인
mysql -uroot -padmin

# source DB 생성
create database video_games;

use video_games;

# video_games 안에 테이블 생성
CREATE TABLE vgsales (
  RankValue INT,
  Name VARCHAR(200),
  Platform VARCHAR(20),
  Year INT,
  Genre VARCHAR(50),
  Publisher VARCHAR(100),
  NA_Sales DECIMAL(10,2),
  EU_Sales DECIMAL(10,2),
  JP_Sales DECIMAL(10,2),
  Other_Sales DECIMAL(10,2),
  Global_Sales DECIMAL(10,2),
  PRIMARY KEY (RankValue)
);

docker cp vgame_sales.csv mysql-sink:/var/lib/mysql-files

LOAD DATA INFILE '/var/lib/mysql-files/vgame_sales.csv' INTO TABLE vgsales FIELDS TERMINATED BY ',' ENCLOSED BY '"' LINES TERMINATED BY '\\n' IGNORE 1 ROWS;

# mysqluser에게 모든 데이터베이스, 테이블 권한 부여
use mysql;
select host,user from user;
grant all privileges on *.* to 'mysqluser'@'%';
FLUSH PRIVILEGES;

Untitled