update

a9b697c5 · 徐泽意 · a9b697c5 · a9b697c5 · a9b697c5 · a9b697c5
Commit a9b697c5 authored Mar 20, 2020 by 徐泽意
12 changed files
--- a/.helmignore
+++ b/.helmignore
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+# Images
+image/
\ No newline at end of file
--- a/Chart.yaml
+++ b/Chart.yaml
+apiVersion: v1
+name: hiveserver
+version: 0.1.0
+description: HiveServer for Kubernetes, modified from Datalake
--- a/image/.gitignore
+++ b/image/.gitignore
+*.tar
+*.tar.gz
+*.tar.bz2
+*.jar
--- a/image/Dockerfile
+++ b/image/Dockerfile
+FROM chenseanxy/hbase:2.1.7-hadoop3.1.2
+ARG VERSION=
+ADD apache-hive-${VERSION}-bin.tar.gz /opt
+RUN cd /opt && ln -s ./apache-hive-${VERSION}-bin hive && \
+    rm -rf /opt/hive/lib/guava* && \ 
+    cp /usr/local/hadoop/share/hadoop/common/lib/guava* /opt/hive/lib && \
+    rm -rf /opt/hive/lib/hbase* && \
+    cp /opt/hbase/lib/hbase* /opt/hive/lib && \
+    .
+ADD mysql-connector-java-8.0.18.jar /opt/hive/lib
+WORKDIR /opt/hive
--- a/image/Makefile
+++ b/image/Makefile
+.PHONY: build
+# Current apache hive releases: http://hive.apache.org/releases.html
+HBASE_VERSION = 2.1.7
+HIVE_VERSION = 3.1.2
+DOCKER = docker
+DOCKER_REPO = chenseanxy/hive
+all: build
+### Build hive versioned images with native libs.
+build:
+	$(DOCKER) build -t hive:$(HIVE_VERSION)-hbase$(HBASE_VERSION) \
+	--build-arg VERSION=$(HIVE_VERSION) \
+	.
+tag: 
+	$(DOCKER) tag \
+	hive:$(HIVE_VERSION)-hbase$(HBASE_VERSION) \
+	$(DOCKER_REPO):$(HIVE_VERSION)-hbase$(HBASE_VERSION)
+push: 
+	$(DOCKER) push $(DOCKER_REPO):$(HIVE_VERSION)-hbase$(HBASE_VERSION)
+### Fetch binary distribution
+apache-hive-%-bin.tar.gz:
+	curl -LO http://archive.apache.org/dist/hive/hive-$(HIVE_VERSION)/apache-hive-$(HIVE_VERSION)-bin.tar.gz
+### TODO: Fetch mysql java connector here
+mysql-java-connector.jar:
+	echo "Please download mysql java connector"
+# Keep intermediate downloads.
+.PRECIOUS: apache-hive-%-bin.tar.gz
--- a/image/README.md
+++ b/image/README.md
+# HBase Image
+Original Image: [PierreZ/hbase-docker](https://github.com/PierreZ/hbase-docker/tree/master/distributed-1.3.1-hadoop-2.7.3)
+Current Version: 1.4.10 based on Hadoop 3.1.2
+Docker Hub: chenseanxy/hbase
--- a/templates/_helpers.tpl
+++ b/templates/_helpers.tpl
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "hive.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "hive.fullname" -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "hive.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
--- a/templates/configmap.yaml
+++ b/templates/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "hive.fullname" . }}
+  labels:
+    app: {{ include "hive.name" . }}
+    chart: {{ include "hive.chart" . }}
+    release: {{ .Release.Name }}
+data:
+  hive-site.xml: |
+    <configuration>
+      <property>
+        <name>hive.server2.transport.mode</name>
+        <value>{{ .Values.Hiveserver2.TransportMode }}</value>
+      </property>
+        <property>
+        <name>hive.server2.thrift.http.port</name>
+        <value>10001</value>
+       </property>
+       <property>
+        <name>hive.server2.thrift.port</name>
+        <value>10001</value>
+      </property>
+      <property>
+        <name>fs.default.name</name>
+        <value>hdfs://{{ .Values.connect.hdfs.namenode }}</value>
+      </property>
+      <property>
+        <name>hive.server2.thrift.max.worker.threads</name>
+        <value>500</value>
+      </property>
+      <property>
+        <name>hive.metastore.event.db.notification.api.auth</name>
+        <value>false</value>
+      </property>
+      <property>
+          <name>hive.metastore.uris</name>
+          <value />
+      </property>  
+      <property>
+          <name>hive.metastore.batch.retrieve.max</name>
+          <value>300</value>
+      </property>
+      <property>
+          <name>hive.metastore.batch.retrieve.table.partition.max</name>
+          <value>1000</value>
+      </property>
+      <property>
+          <name>javax.jdo.option.ConnectionURL</name>
+          <value>jdbc:mysql://{{ .Values.connect.mysql.server }}/hive?createDatabaseIfNotExist=true&amp;useSSL=false</value>
+      </property>
+      <property>
+          <name>javax.jdo.option.ConnectionDriverName</name>
+          <value>com.mysql.jdbc.Driver</value>
+      </property>
+      <property>
+          <name>javax.jdo.option.ConnectionUserName</name>
+          <value>{{ .Values.connect.mysql.username }}</value>
+      </property>
+      <property>
+          <name>javax.jdo.option.ConnectionPassword</name>
+          <value>{{ .Values.connect.mysql.password }}</value>
+      </property>
+      <property>
+          <name>hive.metastore.schema.verification</name>
+          <value>false</value>
+      </property>
+      <property>
+        <name>hive.server2.thrift.client.user</name>
+        <value>mysql</value>
+        <description>Username to use against thrift client</description>
+      </property>
+      <property>
+        <name>hive.server2.thrift.client.password</name>
+        <value>spaceIN511</value>
+        <description>Password to use against thrift client</description>
+      </property>
+      <property>
+          <name>hbase.zookeeper.quorum</name>
+          <value>{{ .Values.connect.hbase.zookeeper }}</value>
+      </property>
+    </configuration>
+  bootstrap.sh: |
+    export PATH=/opt/hive/bin:$PATH
+    HADOOP_HOME=/usr/local/hadoop
+    echo "Schema Init:"
+    schematool -dbType mysql -initSchema
+    hive --service hiveserver2 --hiveconf hive.root.logger=INFO,console
--- a/templates/deployment.yaml
+++ b/templates/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "hive.fullname" . }}
+  labels:
+    app: {{ include "hive.name" . }}
+    chart: {{ include "hive.chart" . }}
+    release: {{ .Release.Name }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ include "hive.name" . }}
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: {{ include "hive.name" . }}
+        release: {{ .Release.Name }}
+    spec:
+      hostname: hiveserver
+      containers:
+        - name: {{ .Chart.Name }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command:
+           - "/bin/bash"
+           - "/tmp/bootstrap.sh"
+          ports:
+            - name: http
+              containerPort: 10001
+              protocol: TCP
+            - name: ui
+              containerPort: 10002
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /
+              port: ui
+            initialDelaySeconds: 100
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /
+              port: ui
+            initialDelaySeconds: 100
+            timeoutSeconds: 5
+          volumeMounts:
+            - name: hive-tmp
+              mountPath: /home
+            - name: hive-config
+              mountPath: /opt/hive/conf/hive-site.xml
+              subPath: hive-site.xml
+              readOnly: true
+            - name: bootstrap
+              mountPath: /tmp/bootstrap.sh
+              subPath: bootstrap.sh
+              readOnly: true
+          resources:
+{{ toYaml .Values.resources | indent 12 }}
+    {{- with .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+      volumes:
+        - name: hive-tmp
+          hostPath:
+            path: /data/hivetmp
+        - name: hive-config
+          configMap:
+            name: {{ include "hive.fullname" . }}
+        - name: bootstrap
+          configMap:
+            name: {{ include "hive.fullname" . }}
--- a/templates/service.yaml
+++ b/templates/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "hive.fullname" . }}-http
+  labels:
+    app: {{ include "hive.name" . }}
+    chart: {{ include "hive.chart" . }}
+    release: {{ .Release.Name }}
+spec:
+  clusterIP: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.portHTTP }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app: {{ include "hive.name" . }}
+    release: {{ .Release.Name }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "hive.fullname" . }}-ui
+  labels:
+    app: {{ include "hive.name" . }}
+    chart: {{ include "hive.chart" . }}
+    release: {{ .Release.Name }}
+spec:
+  type: NodePort
+  ports:
+    - port: {{ .Values.service.portUI }}
+      targetPort: ui
+      protocol: TCP
+      nodePort: {{ .Values.service.nodePortUI }}
+      name: ui
+  selector:
+    app: {{ include "hive.name" . }}
+    release: {{ .Release.Name }}
--- a/values-overrides.yaml
+++ b/values-overrides.yaml
+replicaCount: 1
+image:
+    repository: registry.cn-qingdao.aliyuncs.com/wod/hive
+    tag: 3.1.2
+    pullPolicy: IfNotPresent
+service:
+    type: None
+    portHTTP: 10001
+    portUI: 10002
+    nodePortUI: 30004
+    nodePortHTTP: 30005
+Hiveserver2:
+    TransportMode: binary
+## Server transport mode. "binary" or "http"
+ingress:
+    enabled: true
+resources: {}
+nodeSelector: {}
+tolerations: []
+affinity: {}
+connect:
+    hdfs: 
+        namenode: hadoop-hadoop-hdfs-nn:9000
+        path: /hive/warehouse
+    mysql:
+        server: hive-mysql-mysqlha-readonly:3306
+        username: root
+        password: spaceIN511
+    hbase:
+        zookeeper: myzk-zookeeper:2181
--- a/values.yaml
+++ b/values.yaml
+replicaCount: 1
+image:
+    repository: chenseanxy/hive
+    tag: 3.1.2-hbase2.1.7
+    pullPolicy: IfNotPresent
+service:
+    type: None
+    portHTTP: 10001
+    portUI: 10002
+    nodePortUI: 30004
+    nodePortHTTP: 30005
+ingress:
+    enabled: false
+resources: {}
+nodeSelector: {}
+tolerations: []
+affinity: {}
+connect:
+    hdfs: 
+        namenode: hadoop-hdfs-nn:9000
+        path: /hive/warehouse
+    mysql:
+        server: mysql:3306
+        username: root
+        password: notproduction
+    hbase:
+        zookeeper: zookeeper:2181
\ No newline at end of file