Skip to content

Commit 958dfd0

Browse files
caseylittoncaseylitton
committed
refactor
1 parent 263382a commit 958dfd0

File tree

3 files changed

+86
-63
lines changed

3 files changed

+86
-63
lines changed

‎src/snovault/dev_servers.py‎

Lines changed: 8 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323

2424

2525
EPILOG = __doc__
26-
ENCD_CONFIG_DIR = os.environ.get('ENCD_CONFIG_DIR')
26+
2727
logger = logging.getLogger(__name__)
2828

2929

@@ -60,13 +60,12 @@ def main():
6060
parser.add_argument('--init', action="store_true", help="Init database")
6161
parser.add_argument('--load', action="store_true", help="Load test set")
6262
parser.add_argument('--datadir', default='/tmp/snovault', help="path to datadir")
63-
parser.add_argument('-c', '--encd-config-dir', default=ENCD_CONFIG_DIR, help="ES dir with config")
6463
parser.add_argument(
6564
'-e',
66-
'--es-config-version',
65+
'--es-version',
6766
default=5,
6867
type=int,
69-
help="ES dir with config"
68+
help='Override defult elasticsearch_fixture es version'
7069
)
7170
args = parser.parse_args()
7271

@@ -85,37 +84,18 @@ def main():
8584
from snovault.elasticsearch import create_mapping
8685
datadir = os.path.abspath(args.datadir)
8786
pgdata = os.path.join(datadir, 'pgdata')
88-
esdata = os.path.join(datadir, 'data')
89-
es_data_dir = None
90-
esconfig = None
91-
if args.encd_config_dir:
92-
es_data_dir = f"{ENCD_CONFIG_DIR}/elasticsearch/es{args.es_config_version}"
93-
es_config_dir = f"{es_data_dir}/config"
94-
es_yml_path = f"{es_config_dir}/elasticsearch.yml"
95-
if os.path.exists(es_yml_path):
96-
# es yml config overrides logs/data/host/port vars
97-
esconfig = es_config_dir
98-
esdata = es_data_dir
9987
redisdata = os.path.join(datadir, 'redisdata')
10088
if args.clear:
101-
rm_dirs = [args.datadir]
102-
if esconfig:
103-
rm_dirs.extend([f"{es_data_dir}/data", f"{es_data_dir}/logs", f"{esconfig}/scripts"])
104-
es_keystore_path = f"{esconfig}/elasticsearch.keystore"
105-
if os.path.exists(es_keystore_path):
106-
os.remove(es_keystore_path)
107-
for dirname in rm_dirs:
108-
if os.path.exists(dirname):
109-
shutil.rmtree(dirname)
89+
if os.path.exists(args.datadir):
90+
shutil.rmtree(args.datadir)
11091
if args.init:
11192
postgresql_fixture.initdb(pgdata, echo=True)
112-
11393
postgres = postgresql_fixture.server_process(pgdata, echo=True)
11494
elasticsearch = elasticsearch_fixture.server_process(
115-
esdata,
116-
configdir=esconfig,
95+
datadir,
96+
clear=args.clear,
11797
echo=True,
118-
version=args.es_config_version,
98+
version=args.es_version,
11999
)
120100
nginx = nginx_server_process(echo=True)
121101
redis_config_path = redis_storage_fixture.initdb(redisdata, local_storage_port, echo=True)

‎src/snovault/tests/elasticsearch_fixture.py‎

Lines changed: 77 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import os.path
2+
import shutil
23
import sys
34
from time import sleep
45
try:
@@ -8,42 +9,84 @@
89

910

1011
# These are set in .circleci/config
11-
CIRCLE_BUILD = os.environ.get('BASH_ENV') == '/home/circleci/.bashrc'
12-
CICLE_ES_VERSION = os.environ.get('ES_MAJOR_VERSION')
12+
_CIRCLE_BUILD = os.environ.get('BASH_ENV') == '/home/circleci/.bashrc'
13+
_CICLE_ES_VERSION = os.environ.get('ES_MAJOR_VERSION')
14+
# Encoded configuraion repo
15+
_ENCD_CONFIG_DIR = os.environ.get('ENCD_CONFIG_DIR')
16+
# This is required for local osx testing, circle version is sepearte
17+
_OSX_ES_VERSION = 5
1318

1419

15-
def _get_args_and_env():
16-
pass
17-
18-
19-
def server_process(
20-
datadir,
21-
configdir=None,
22-
echo=False,
23-
host='127.0.0.1',
24-
port=9201,
25-
prefix='',
26-
version=5,
27-
):
20+
def _get_args_and_env(esdata, esdata_override, kwargs):
2821
env = os.environ.copy()
29-
if configdir:
30-
# How elasticsearch 6+ sets config path
31-
env["ES_PATH_CONF"] = configdir
32-
args = [
33-
'elasticsearch',
34-
]
35-
if version <= 5:
22+
args = [
23+
os.path.join(kwargs.get('prefix', ''), 'elasticsearch'),
24+
]
25+
if esdata_override:
26+
# ES configuration dir is specified, not default location.
27+
# 'esdata' is ignored and kwargs values are overidden by elasticsearch.yml
28+
esconfig = f"{esdata_override}/config"
29+
if kwargs['version'] <= 5:
3630
# How elasticsearch 5 sets config path
37-
args.append(f"-Epath.conf={configdir}")
38-
else:
39-
args = [
40-
os.path.join(prefix, 'elasticsearch'),
41-
'-Enetwork.host=%s' % host,
42-
'-Ehttp.port=%d' % port,
43-
'-Epath.data=%s' % os.path.join(datadir, 'data'),
44-
'-Epath.logs=%s' % os.path.join(datadir, 'logs'),
45-
'-Epath.conf=./conf',
46-
]
31+
args.append(f"-Epath.conf={esconfig}")
32+
else:
33+
# How elasticsearch 6+ sets config path
34+
env["ES_PATH_CONF"] = esconfig
35+
jvm_options = _get_jvm_options(f"{esconfig}/jvm.options")
36+
if jvm_options:
37+
env['ES_JAVA_OPTS'] = jvm_options
38+
return args, env
39+
# Default 'esdata' location
40+
args.extend([
41+
'-Enetwork.host=%s' % kwargs.get('host', '127.0.0.1'),
42+
'-Ehttp.port=%d' % kwargs.get('port', 9201),
43+
'-Epath.data=%s' % os.path.join(esdata, 'data'),
44+
'-Epath.logs=%s' % os.path.join(esdata, 'logs'),
45+
])
46+
if _CIRCLE_BUILD and int(_CICLE_ES_VERSION) == 5:
47+
args.append('-Epath.conf=./conf')
48+
return args, env
49+
50+
51+
def _get_config_override(esdata, kwargs):
52+
if _ENCD_CONFIG_DIR:
53+
# If found, elasticsearch.yml config overrides vars like logs/data/host/port/etc...
54+
es_data_dir = f"{_ENCD_CONFIG_DIR}/elasticsearch/es{kwargs['version']}"
55+
if kwargs.get('local_test', False):
56+
es_data_dir += 'test'
57+
if os.path.exists(f"{es_data_dir}/config/elasticsearch.yml"):
58+
return es_data_dir
59+
return None
60+
61+
62+
def _get_jvm_options(jvm_options_path):
63+
with open(jvm_options_path, 'r') as fileh:
64+
return ' '.join([
65+
line.strip()
66+
for line in fileh.readlines()
67+
])
68+
69+
70+
def _clear(esdata, esdata_override):
71+
rm_dirs = [esdata]
72+
if esdata_override:
73+
esconfig = f"{esdata_override}/config"
74+
rm_dirs.extend([f"{esdata_override}/data", f"{esdata_override}/logs", f"{esconfig}/scripts"])
75+
es_keystore_path = f"{esconfig}/elasticsearch.keystore"
76+
if os.path.exists(es_keystore_path):
77+
os.remove(es_keystore_path)
78+
for dir_path in rm_dirs:
79+
if os.path.exists(dir_path):
80+
shutil.rmtree(dir_path)
81+
82+
83+
def server_process(datadir, **kwargs):
84+
kwargs['version'] = kwargs.get('version', _OSX_ES_VERSION)
85+
esdata = os.path.join(datadir, 'data')
86+
esdata_override = _get_config_override(esdata, kwargs)
87+
if kwargs.get('clear', False):
88+
_clear(esdata, esdata_override)
89+
args, env = _get_args_and_env(esdata, esdata_override, kwargs)
4790
process = subprocess.Popen(
4891
args,
4992
close_fds=True,
@@ -55,7 +98,7 @@ def server_process(
5598
SUCCESS_LINE = b'started\n'
5699
lines = []
57100
for line in iter(process.stdout.readline, b''):
58-
if echo:
101+
if kwargs.get('echo', False):
59102
sys.stdout.write(line.decode('utf-8'))
60103
lines.append(line)
61104
if line.endswith(SUCCESS_LINE):
@@ -66,7 +109,7 @@ def server_process(
66109
msg = ('Process return code: %d\n' % code) + b''.join(lines).decode('utf-8')
67110
raise Exception(msg)
68111

69-
if not echo:
112+
if not kwargs.get('echo', False):
70113
process.stdout.close()
71114
print('returning process')
72115
return process

‎src/snovault/tests/serverfixtures.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ def elasticsearch_server(request, elasticsearch_host_port):
7474
host, port = elasticsearch_host_port
7575
tmpdir = request.config._tmpdirhandler.mktemp('elasticsearch', numbered=True)
7676
tmpdir = str(tmpdir)
77-
process = server_process(str(tmpdir), host=host, port=9201, echo=False)
77+
process = server_process(str(tmpdir), local_test=True, host=host, port=9201, echo=False)
7878
print('PORT CHANGED')
7979
yield 'http://%s:%d' % (host, 9201)
8080

0 commit comments

Comments
 (0)