r4906 - trunk/test/stress-test
daigle at ecoinformatics.org
daigle at ecoinformatics.org
Wed Apr 8 10:59:43 PDT 2009
Author: daigle
Date: 2009-04-08 10:59:43 -0700 (Wed, 08 Apr 2009)
New Revision: 4906
Added:
trunk/test/stress-test/generate-dictionaries.sh
trunk/test/stress-test/read-load-test.py
Modified:
trunk/test/stress-test/insert-load-test.py
trunk/test/stress-test/load-test-driver.sh
trunk/test/stress-test/metacat.py
trunk/test/stress-test/squery-load-test.py
Log:
Update load test code
Added: trunk/test/stress-test/generate-dictionaries.sh
===================================================================
--- trunk/test/stress-test/generate-dictionaries.sh (rev 0)
+++ trunk/test/stress-test/generate-dictionaries.sh 2009-04-08 17:59:43 UTC (rev 4906)
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+#############################################################################
+# Create dictionary files of five letter words with every possible combination
+# of letters. Each file is named dictionary-X.txt where X is each letter of
+# the alphabet and the file holds all possible words that start with that
+# letter.
+#
+# 6 April 2009 Michael Daigle (daigle at nceas.ucsb.edu)
+#############################################################################
+
+ltr=(a b c d e f g h i j k l m n o p q r s t u v w x y z)
+
+> ./dictionary2.txt
+
+for i in {0..25}
+do
+ for j in {0..25}
+ do
+ for k in {0..25}
+ do
+ for l in {0..25}
+ do
+ for m in {0..25}
+ do
+ echo ${ltr[$i]}${ltr[$j]}${ltr[$k]}${ltr[$l]}${ltr[$m]} >> ./dictionary-${ltr[$i]}.txt
+ done
+ done
+ done
+ done
+done
\ No newline at end of file
Property changes on: trunk/test/stress-test/generate-dictionaries.sh
___________________________________________________________________
Name: svn:executable
+ *
Modified: trunk/test/stress-test/insert-load-test.py
===================================================================
--- trunk/test/stress-test/insert-load-test.py 2009-04-06 22:28:46 UTC (rev 4905)
+++ trunk/test/stress-test/insert-load-test.py 2009-04-08 17:59:43 UTC (rev 4906)
@@ -11,38 +11,49 @@
interval = int(sys.argv[3])
host = sys.argv[4]
-# set debug=0 for no debugging. Set it to 0 - 3 for increasing amounts of debug.
-debug=0
+# set debug_level=0 for no debugging. Set it to 0 - 3 for increasing amounts of debug.
+debug_level=2
run_time = str(time()).replace('.','')
-output_file = "./insert-" + letter + ".out"
-output_file = open(output_file, 'w')
+output_file_name = "./insert-" + letter + ".out"
+output_file = open(output_file_name, 'w')
+output_file.close()
def debug (message):
+ output_file = open(output_file_name, 'a')
output_file.write(message + '\n')
+ output_file.close()
def debug1 (message):
- if debug > 0:
+ if debug_level > 0:
+ output_file = open(output_file_name, 'a')
output_file.write(message + '\n')
+ output_file.close()
def debug2 (message):
- if debug > 1:
+ if debug_level > 1:
+ output_file = open(output_file_name, 'a')
output_file.write(message + '\n')
+ output_file.close()
def debug3 (message):
- if debug > 2:
+ if debug_level > 2:
+ output_file = open(output_file_name, 'a')
output_file.write(message + '\n')
+ output_file.close()
-dictionary_file = open('./dictionary.txt', 'r')
+dictionary_file_name = './dictionary-' + letter + '.txt'
+dictionary_file = open(dictionary_file_name, 'r')
iter_count = 0
debug("************************************************** ")
-debug("Starting squery-load-test for ")
+debug("Starting insert-load-test for ")
debug(" letter: $letter ")
debug(" iterations: $iterations ")
debug(" interval: $interval ")
+debug(" host: $host ")
debug("************************************************** ")
# Initialize a metacat client connection and log in as test user
@@ -62,31 +73,30 @@
insert_template_file.close
for line in dictionary_file:
- if line.lower().startswith(letter.lower()):
- word = line.strip()
+ word = line.strip()
- debug3("word: " + word)
- debug3("run time: " + run_time)
- debug3("iter count: " + str(iter_count))
- docid = word + run_time + '.' + str(iter_count) + '.1'
- doc = insert_template.replace("@!docid!@", docid)
- t1 = time()
+ debug3("word: " + word)
+ debug3("run time: " + run_time)
+ debug3("iter count: " + str(iter_count))
+ docid = word + run_time + '.' + str(iter_count) + '.1'
+ doc = insert_template.replace("@!docid!@", docid)
+ t1 = time()
- debug("[%s] -- Processing Insert" % (docid))
- response = mc.insert(docid, doc)
- t2 = time()
+ debug("[%s] -- Processing Insert" % (docid))
+ response = mc.insert(docid, doc)
+ t2 = time()
- if (response.lower().find('<error>') == -1):
- debug("[%s] -- SUCCESS: elapsed time: %.5f seconds" % (docid, t2-t1))
- else:
- debug("[%s] -- ERROR: %s" % (docid, response))
+ if (response.lower().find('<error>') == -1):
+ debug("[%s] -- SUCCESS: elapsed time: %.5f seconds" % (docid, t2-t1))
+ else:
+ debug("[%s] -- ERROR: %s" % (docid, response))
- iter_count = iter_count + 1
- debug3("iter_count: %d, iterations: %d" % (iter_count, iterations))
- if iter_count >= iterations:
- sys.exit()
+ iter_count = iter_count + 1
+ debug2("iter_count: %d, iterations: %d" % (iter_count, iterations))
+ if iter_count >= iterations:
+ sys.exit()
- sleep(interval)
+ sleep(interval)
dictionary_file.close()
output_file.close()
\ No newline at end of file
Modified: trunk/test/stress-test/load-test-driver.sh
===================================================================
--- trunk/test/stress-test/load-test-driver.sh 2009-04-06 22:28:46 UTC (rev 4905)
+++ trunk/test/stress-test/load-test-driver.sh 2009-04-08 17:59:43 UTC (rev 4906)
@@ -6,18 +6,55 @@
# 3 April 2009 Michael Daigle (daigle at nceas.ucsb.edu)
#############################################################################
+######################
# squery-load-test.py dictionary-letter iterations delay test-server
+######################
##### run squeries #####
-./squery-load-test.py a 100 3 dev.nceas.ucsb.edu 2>&1 &
+#./squery-load-test.py m 50 2 dev.nceas.ucsb.edu 2>&1 &
##### run squeries #####"
-./squery-load-test.py b 100 3 dev.nceas.ucsb.edu 2>&1 &
+#./squery-load-test.py n 50 2 dev.nceas.ucsb.edu 2>&1 &
##### run squeries #####
-./squery-load-test.py c 100 3 dev.nceas.ucsb.edu 2>&1 &
+#./squery-load-test.py o 50 2 dev.nceas.ucsb.edu 2>&1 &
##### run squeries #####
-./squery-load-test.py d 100 3 dev.nceas.ucsb.edu 2>&1 &
+#./squery-load-test.py p 50 2 dev.nceas.ucsb.edu 2>&1 &
+######################
+# insert-load-test.py dictionary-letter iterations delay test-server
+######################
+##### run squeries #####
+./insert-load-test.py a 50 2 dev.nceas.ucsb.edu 2>&1 &
+
+##### run squeries #####"
+./insert-load-test.py b 50 2 dev.nceas.ucsb.edu 2>&1 &
+
+##### run squeries #####
+./insert-load-test.py c 50 2 dev.nceas.ucsb.edu 2>&1 &
+
+##### run squeries #####
+./insert-load-test.py d 50 2 dev.nceas.ucsb.edu 2>&1 &
+
+######################
+# read-load-test.py prefix iterations delay test-server
+######################
+
+##### run squeries #####
+#./read-load-test.py a 300 3 dev.nceas.ucsb.edu 2>&1 &
+
+##### run squeries #####"
+#./read-load-test.py b 300 3 dev.nceas.ucsb.edu 2>&1 &
+
+##### run squeries #####
+#./read-load-test.py c 300 3 dev.nceas.ucsb.edu 2>&1 &
+
+##### run squeries #####
+#./read-load-test.py d 300 3 dev.nceas.ucsb.edu 2>&1 &
+
+echo
+echo "Running the following load tests:"
+ps auxwww |grep load-test.py |grep -v grep
+
Modified: trunk/test/stress-test/metacat.py
===================================================================
--- trunk/test/stress-test/metacat.py 2009-04-06 22:28:46 UTC (rev 4905)
+++ trunk/test/stress-test/metacat.py 2009-04-08 17:59:43 UTC (rev 4906)
@@ -71,15 +71,12 @@
return False
def read(self, docid, qformat="xml"):
- postdata = { 'action' : 'read',
+ postdata = { 'action' : 'read',
'qformat' : qformat,
'docid' : docid }
- response = self.postRequest(postdata)
- # if error node returned
- if response.find("<error>") != -1:
- return False
- else:
- return response
+ response = self.postRequest(postdata)
+ # if error node returned
+ return response
def insert(self, docid, doctext):
Added: trunk/test/stress-test/read-load-test.py
===================================================================
--- trunk/test/stress-test/read-load-test.py (rev 0)
+++ trunk/test/stress-test/read-load-test.py 2009-04-08 17:59:43 UTC (rev 4906)
@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+
+import metacat
+import sys
+
+from time import time
+from time import sleep
+
+prefix = sys.argv[1]
+iterations = int(sys.argv[2])
+interval = int(sys.argv[3])
+host = sys.argv[4]
+
+# set debug_level=0 for no debugging. Set it to 0 - 3 for increasing amounts of debug.
+debug_level=2
+
+run_time = str(time()).replace('.','')
+
+output_file_name = "./read-" + prefix + ".out"
+output_file = open(output_file_name, 'w')
+output_file.close()
+
+def debug (message):
+ output_file = open(output_file_name, 'a')
+ output_file.write(message + '\n')
+ output_file.close()
+
+def debug1 (message):
+ if debug_level > 0:
+ output_file = open(output_file_name, 'a')
+ output_file.write(message + '\n')
+ output_file.close()
+
+def debug2 (message):
+ if debug_level > 1:
+ output_file = open(output_file_name, 'a')
+ output_file.write(message + '\n')
+ output_file.close()
+
+def debug3 (message):
+ if debug_level > 2:
+ output_file = open(output_file_name, 'a')
+ output_file.write(message + '\n')
+ output_file.close()
+
+iter_count = 0
+
+debug("************************************************** ")
+debug("Starting read-load-test for ")
+debug(" letter: $letter ")
+debug(" iterations: $iterations ")
+debug(" interval: $interval ")
+debug(" host: $host ")
+debug("************************************************** ")
+
+# Initialize a metacat client connection and log in as test user
+t1 = time()
+mc = metacat.MetacatClient(host,'/knb/metacat')
+debug("[test] -- Processing Login")
+response = mc.login('test', 'test', 'NCEAS')
+t2 = time()
+
+if (response):
+ debug("[test] -- SUCCESS: elapsed time: %.5f seconds" % (t2-t1))
+else:
+ debug("[test] -- ERROR: could not log in")
+
+# Insert a document
+insert_template_file = open('insert.xml.tmpl', 'r')
+insert_template = insert_template_file.read()
+insert_template_file.close
+
+t1 = time()
+docid = 'readtest-' + prefix + run_time + '.1.1'
+doc = insert_template.replace("@!docid!@", docid)
+debug("[%s] -- Processing Insert" % (docid))
+response = mc.insert(docid, doc)
+t2 = time()
+
+if (response.lower().find('<error>') == -1):
+ debug("[%s] -- SUCCESS: elapsed time: %.5f seconds" % (docid, t2-t1))
+else:
+ debug("[%s] -- ERROR: %s" % (docid, response))
+t2 = time()
+
+for i in range(1,iterations):
+ t1 = time()
+
+ debug("[%s] -- Processing Read" % (docid))
+ response = mc.read(docid)
+ t2 = time()
+
+ if (response.lower().find('<error>') == -1):
+ debug("[%s] -- SUCCESS: elapsed time: %.5f seconds" % (docid, t2-t1))
+ else:
+ debug("[%s] -- ERROR: %s" % (docid, response))
+
+ iter_count = iter_count + 1
+ debug2("iter_count: %d, iterations: %d" % (iter_count, iterations))
+
+ sleep(interval)
+
+output_file.close()
\ No newline at end of file
Modified: trunk/test/stress-test/squery-load-test.py
===================================================================
--- trunk/test/stress-test/squery-load-test.py 2009-04-06 22:28:46 UTC (rev 4905)
+++ trunk/test/stress-test/squery-load-test.py 2009-04-08 17:59:43 UTC (rev 4906)
@@ -11,28 +11,38 @@
interval = int(sys.argv[3])
host = sys.argv[4]
-# set debug=0 for no debugging. Set it to 0 - 3 for increasing amounts of debug.
-debug=0
+# set debug_level=0 for no debugging. Set it to 0 - 3 for increasing amounts of debug.
+debug_level=2
-output_file = "./squery-" + letter + ".out"
-output_file = open(output_file, 'w')
+output_file_name = "./squery-" + letter + ".out"
+output_file = open(output_file_name, 'w')
+output_file.close()
def debug (message):
+ output_file = open(output_file_name, 'a')
output_file.write(message + '\n')
+ output_file.close()
def debug1 (message):
- if debug > 0:
+ if debug_level > 0:
+ output_file = open(output_file_name, 'a')
output_file.write(message + '\n')
+ output_file.close()
def debug2 (message):
- if debug > 1:
+ if debug_level > 1:
+ output_file = open(output_file_name, 'a')
output_file.write(message + '\n')
+ output_file.close()
def debug3 (message):
- if debug > 2:
+ if debug_level > 2:
+ output_file = open(output_file_name, 'a')
output_file.write(message + '\n')
+ output_file.close()
-dictionary_file = open('./dictionary.txt', 'r')
+dictionary_file_name = './dictionary-' + letter + '.txt'
+dictionary_file = open(dictionary_file_name, 'r')
iter_count = 0
@@ -41,6 +51,7 @@
debug(" letter: $letter ")
debug(" iterations: $iterations ")
debug(" interval: $interval ")
+debug(" host: $host ")
debug("************************************************** ")
mc = metacat.MetacatClient(host,'/knb/metacat')
@@ -50,25 +61,24 @@
query_template_file.close
for line in dictionary_file:
- if line.lower().startswith(letter.lower()):
- word = line.strip()
- debug("[%s] -- Processing Search" % (word))
- query = query_template.replace("@!search-word!@", word)
- t1 = time()
- response = mc.squery(query)
- t2 = time()
+ word = line.strip()
+ debug("[%s] -- Processing Search" % (word))
+ query = query_template.replace("@!search-word!@", word)
+ t1 = time()
+ response = mc.squery(query)
+ t2 = time()
- if response.find("<resultset>") != -1:
- debug("[%s] -- SUCCESS: elapsed time: %.5f seconds" % (word, t2-t1))
- else:
- debug("[%s] -- ERROR: " % (word), response)
+ if response.find("<resultset>") != -1:
+ debug("[%s] -- SUCCESS: elapsed time: %.5f seconds" % (word, t2-t1))
+ else:
+ debug("[%s] -- ERROR: " % (word), response)
- iter_count = iter_count + 1
- debug3("iter_count: %d, iterations: %d" % (iter_count, iterations))
- if iter_count >= iterations:
- sys.exit()
+ iter_count = iter_count + 1
+ debug2("iter_count: %d, iterations: %d" % (iter_count, iterations))
+ if iter_count >= iterations:
+ sys.exit()
- sleep(interval)
+ sleep(interval)
dictionary_file.close()
output_file.close()
More information about the Metacat-cvs
mailing list