rtyler
/
Roxanne
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

modularizing, implemented new protocol

This commit is contained in:
Joseph Rothrock 2011-12-30 15:01:33 -08:00
parent 8963241b31
commit e232be2ab5
9 changed files with 762 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Makefile
View File

@ -1,8 +1,12 @@
OS := $(shell uname)
ifeq (${OS},Linux)
libs = -lrt -lm
endif
default: dbr
dbr: roxanne_db.c
gcc -Werror -g hash_32.c roxanne_db.c -o dbr
gcc -Werror -g hash_32.c roxanne_db.c -o dbr $(libs)
chmod 755 dbr
.PHONY: clean

51
README
View File

@ -7,8 +7,6 @@ To install it:
$ sudo make install
This will put the executables in /usr/local/bin
Header file (client lib) goes into /usr/local/include
The client lib (something.so) goes into /usr/local/lib
To initialize or re-create the database files:
@ -18,36 +16,57 @@ To start the db:
$ sudo /usr/local/bin/dbr_ctl start
Look at the top of the dbr_ctl script for some initialization variables you might want to change.
Look at the top of the dbr_ctl script for some initialization variables you
might want to change.
--------------
Precis
--------------
Roxanne is a very simple database server that allows a client to store and retrieve values by key.
Keys are stored in a hash map of 64K buckets. Hash collisions are resolved by separate chaining onto
linked lists at the end of the index file. The default location for the index file is in
Roxanne is a very simple database server that allows a client to store and
retrieve values by key. Keys are stored in a hash map of 64K buckets.
Hash collisions are resolved by separate chaining onto linked lists at the
end of the index file. The default location for the index file is in
/var/roxanne/idx
The values for the given keys are stored in contiguous 4KB blocks in the database file (/var/roxanne/db).
A file called block_bitmap tracks the free/busy blocks in the db file. The dbr processes memory-map this file
for very fast access. The db file starts out with zero blocks. Blocks are only added as needed to
accomodate new records. Constants in the code limit the number of blocks to 134,217,728. This means that
a single Roxanne database instance can store at most this many records.
The values for the given keys are stored in contiguous 4KB blocks in the
database file (/var/roxanne/db). A file called block_bitmap tracks the
free/busy blocks in the db file. The dbr processes memory-map this file
for very fast access. The db file starts out with zero blocks.
Blocks are only added to the db file as needed to accomodate new records.
As typically built, the database can accomodate about a billion blocks.
--------------
Example
--------------
madison:Roxanne rothrock$ sudo dbr_ctl start
Password:
Started listening.
madison:Roxanne rothrock$ telnet localhost 4080
Trying ::1...
Connected to localhost.
Escape character is '^]'.
create: roxanne&A black, German Shepherd mix, about 50#, very smart and very emotional^@
create /alpha/The first record
Write OK.
read: roxanne^@
A black, German Shepherd mix, about 50#, very smart and very emotional
read /alpha
The first record
create /alpha/beta/record
Write OK.
read /alpha/beta
record
delete /alpha
Delete OK.
read /alpha
Not found.
read /alpha/beta
record
--------------
Protocol
--------------
commands:
create /key[/key/key/key...]/value<newline>
read /key[/key/key/key...]
delete /key[/key/key/key...]

12
dbr_ctl
View File

@ -11,7 +11,7 @@ DB_PATH='/var/roxanne'
FILES="block_bitmap idx db"
function usage {
echo "Usage: $0 {start|stop|initdb [force]}"
echo "Usage: $0 {start|stop|kill|initdb [force]}"
echo "This is a run control script used to manage the Roxanne key-value store (dbr)."
exit 1
}
@ -31,11 +31,17 @@ case "$1" in
;;
stop)
test ! -e $PID_DIR/dbr.pid && echo "No pid file found." && exit 1
test ! -e $PID_DIR/dbr.pid && echo "No pid file found. Try 'sudo dbr_ctl kill' instead" && exit 1
sudo -u $RUN_AS_USER kill $(<$PID_DIR/dbr.pid)
sudo rm $PID_DIR/dbr.pid
;;
kill)
sudo killall -u $RUN_AS_USER dbr
sudo rm $PID_DIR/dbr.pid &>/dev/null
;;
initdb)
echo "Path to database files: $DB_PATH"
@ -52,7 +58,7 @@ case "$1" in
chown $RUN_AS_USER:$RUN_AS_GRP $DB_PATH
sudo -u $RUN_AS_USER test ! -w $DB_PATH && echo "Can't write to $DB_PATH" && exit 1
sudo -u $RUN_AS_USER dd if=/dev/zero of=$DB_PATH/block_bitmap bs=1024 count=16384
sudo -u $RUN_AS_USER dd if=/dev/zero of=$DB_PATH/block_bitmap bs=1024 count=131072
sudo -u $RUN_AS_USER dd if=/dev/zero of=$DB_PATH/idx bs=1024 count=65536
sudo -u $RUN_AS_USER cat /dev/null >$DB_PATH/db
chown $RUN_AS_USER:$RUN_AS_GRP $DB_PATH/db

249
fnv.h Executable file
View File

@ -0,0 +1,249 @@
/*
* fnv - Fowler/Noll/Vo- hash code
*
* @(#) $Revision: 5.4 $
* @(#) $Id: fnv.h,v 5.4 2009/07/30 22:49:13 chongo Exp $
* @(#) $Source: /usr/local/src/cmd/fnv/RCS/fnv.h,v $
*
***
*
* Fowler/Noll/Vo- hash
*
* The basis of this hash algorithm was taken from an idea sent
* as reviewer comments to the IEEE POSIX P1003.2 committee by:
*
* Phong Vo (http://www.research.att.com/info/kpv/)
* Glenn Fowler (http://www.research.att.com/~gsf/)
*
* In a subsequent ballot round:
*
* Landon Curt Noll (http://www.isthe.com/chongo/)
*
* improved on their algorithm. Some people tried this hash
* and found that it worked rather well. In an EMail message
* to Landon, they named it the ``Fowler/Noll/Vo'' or FNV hash.
*
* FNV hashes are designed to be fast while maintaining a low
* collision rate. The FNV speed allows one to quickly hash lots
* of data while maintaining a reasonable collision rate. See:
*
* http://www.isthe.com/chongo/tech/comp/fnv/index.html
*
* for more details as well as other forms of the FNV hash.
*
***
*
* NOTE: The FNV-0 historic hash is not recommended. One should use
* the FNV-1 hash instead.
*
* To use the 32 bit FNV-0 historic hash, pass FNV0_32_INIT as the
* Fnv32_t hashval argument to fnv_32_buf() or fnv_32_str().
*
* To use the 64 bit FNV-0 historic hash, pass FNV0_64_INIT as the
* Fnv64_t hashval argument to fnv_64_buf() or fnv_64_str().
*
* To use the recommended 32 bit FNV-1 hash, pass FNV1_32_INIT as the
* Fnv32_t hashval argument to fnv_32_buf() or fnv_32_str().
*
* To use the recommended 64 bit FNV-1 hash, pass FNV1_64_INIT as the
* Fnv64_t hashval argument to fnv_64_buf() or fnv_64_str().
*
* To use the recommended 32 bit FNV-1a hash, pass FNV1_32A_INIT as the
* Fnv32_t hashval argument to fnv_32a_buf() or fnv_32a_str().
*
* To use the recommended 64 bit FNV-1a hash, pass FNV1A_64_INIT as the
* Fnv64_t hashval argument to fnv_64a_buf() or fnv_64a_str().
*
***
*
* Please do not copyright this code. This code is in the public domain.
*
* LANDON CURT NOLL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO
* EVENT SHALL LANDON CURT NOLL BE LIABLE FOR ANY SPECIAL, INDIRECT OR
* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
* USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
* PERFORMANCE OF THIS SOFTWARE.
*
* By:
* chongo <Landon Curt Noll> /\oo/\
* http://www.isthe.com/chongo/
*
* Share and Enjoy! :-)
*/
#if !defined(__FNV_H__)
#define __FNV_H__
#include <sys/types.h>
#define FNV_VERSION "5.0.2" /* @(#) FNV Version */
/*
* 32 bit FNV-0 hash type
*/
typedef u_int32_t Fnv32_t;
/*
* 32 bit FNV-0 zero initial basis
*
* This historic hash is not recommended. One should use
* the FNV-1 hash and initial basis instead.
*/
#define FNV0_32_INIT ((Fnv32_t)0)
/*
* 32 bit FNV-1 and FNV-1a non-zero initial basis
*
* The FNV-1 initial basis is the FNV-0 hash of the following 32 octets:
*
* chongo <Landon Curt Noll> /\../\
*
* NOTE: The \'s above are not back-slashing escape characters.
* They are literal ASCII backslash 0x5c characters.
*
* NOTE: The FNV-1a initial basis is the same value as FNV-1 by definition.
*/
#define FNV1_32_INIT ((Fnv32_t)0x811c9dc5)
#define FNV1_32A_INIT FNV1_32_INIT
/*
* determine how 64 bit unsigned values are represented
*/
#include "longlong.h"
/*
* 64 bit FNV-0 hash
*/
#if defined(HAVE_64BIT_LONG_LONG)
typedef u_int64_t Fnv64_t;
#else /* HAVE_64BIT_LONG_LONG */
typedef struct {
u_int32_t w32[2]; /* w32[0] is low order, w32[1] is high order word */
} Fnv64_t;
#endif /* HAVE_64BIT_LONG_LONG */
/*
* 64 bit FNV-0 zero initial basis
*
* This historic hash is not recommended. One should use
* the FNV-1 hash and initial basis instead.
*/
#if defined(HAVE_64BIT_LONG_LONG)
#define FNV0_64_INIT ((Fnv64_t)0)
#else /* HAVE_64BIT_LONG_LONG */
extern const Fnv64_t fnv0_64_init;
#define FNV0_64_INIT (fnv0_64_init)
#endif /* HAVE_64BIT_LONG_LONG */
/*
* 64 bit FNV-1 non-zero initial basis
*
* The FNV-1 initial basis is the FNV-0 hash of the following 32 octets:
*
* chongo <Landon Curt Noll> /\../\
*
* NOTE: The \'s above are not back-slashing escape characters.
* They are literal ASCII backslash 0x5c characters.
*
* NOTE: The FNV-1a initial basis is the same value as FNV-1 by definition.
*/
#if defined(HAVE_64BIT_LONG_LONG)
#define FNV1_64_INIT ((Fnv64_t)0xcbf29ce484222325ULL)
#define FNV1A_64_INIT FNV1_64_INIT
#else /* HAVE_64BIT_LONG_LONG */
extern const fnv1_64_init;
extern const Fnv64_t fnv1a_64_init;
#define FNV1_64_INIT (fnv1_64_init)
#define FNV1A_64_INIT (fnv1a_64_init)
#endif /* HAVE_64BIT_LONG_LONG */
/*
* hash types
*/
enum fnv_type {
FNV_NONE = 0, /* invalid FNV hash type */
FNV0_32 = 1, /* FNV-0 32 bit hash */
FNV1_32 = 2, /* FNV-1 32 bit hash */
FNV1a_32 = 3, /* FNV-1a 32 bit hash */
FNV0_64 = 4, /* FNV-0 64 bit hash */
FNV1_64 = 5, /* FNV-1 64 bit hash */
FNV1a_64 = 6, /* FNV-1a 64 bit hash */
};
/*
* these test vectors are used as part o the FNV test suite
*/
struct test_vector {
void *buf; /* start of test vector buffer */
int len; /* length of test vector */
};
struct fnv0_32_test_vector {
struct test_vector *test; /* test vector buffer to hash */
Fnv32_t fnv0_32; /* expected FNV-0 32 bit hash value */
};
struct fnv1_32_test_vector {
struct test_vector *test; /* test vector buffer to hash */
Fnv32_t fnv1_32; /* expected FNV-1 32 bit hash value */
};
struct fnv1a_32_test_vector {
struct test_vector *test; /* test vector buffer to hash */
Fnv32_t fnv1a_32; /* expected FNV-1a 32 bit hash value */
};
struct fnv0_64_test_vector {
struct test_vector *test; /* test vector buffer to hash */
Fnv64_t fnv0_64; /* expected FNV-0 64 bit hash value */
};
struct fnv1_64_test_vector {
struct test_vector *test; /* test vector buffer to hash */
Fnv64_t fnv1_64; /* expected FNV-1 64 bit hash value */
};
struct fnv1a_64_test_vector {
struct test_vector *test; /* test vector buffer to hash */
Fnv64_t fnv1a_64; /* expected FNV-1a 64 bit hash value */
};
/*
* external functions
*/
/* hash_32.c */
extern Fnv32_t fnv_32_buf(void *buf, size_t len, Fnv32_t hashval);
extern Fnv32_t fnv_32_str(char *buf, Fnv32_t hashval);
/* hash_32a.c */
extern Fnv32_t fnv_32a_buf(void *buf, size_t len, Fnv32_t hashval);
extern Fnv32_t fnv_32a_str(char *buf, Fnv32_t hashval);
/* hash_64.c */
extern Fnv64_t fnv_64_buf(void *buf, size_t len, Fnv64_t hashval);
extern Fnv64_t fnv_64_str(char *buf, Fnv64_t hashval);
/* hash_64a.c */
extern Fnv64_t fnv_64a_buf(void *buf, size_t len, Fnv64_t hashval);
extern Fnv64_t fnv_64a_str(char *buf, Fnv64_t hashval);
/* test_fnv.c */
extern struct test_vector fnv_test_str[];
extern struct fnv0_32_test_vector fnv0_32_vector[];
extern struct fnv1_32_test_vector fnv1_32_vector[];
extern struct fnv1a_32_test_vector fnv1a_32_vector[];
extern struct fnv0_64_test_vector fnv0_64_vector[];
extern struct fnv1_64_test_vector fnv1_64_vector[];
extern struct fnv1a_64_test_vector fnv1a_64_vector[];
extern void unknown_hash_type(char *prog, enum fnv_type type, int code);
extern void print_fnv32(Fnv32_t hval, Fnv32_t mask, int verbose, char *arg);
extern void print_fnv64(Fnv64_t hval, Fnv64_t mask, int verbose, char *arg);
#endif /* __FNV_H__ */

156
hash_32.c Executable file
View File

@ -0,0 +1,156 @@
/*
* hash_32 - 32 bit Fowler/Noll/Vo hash code
*
* @(#) $Revision: 5.1 $
* @(#) $Id: hash_32.c,v 5.1 2009/06/30 09:13:32 chongo Exp $
* @(#) $Source: /usr/local/src/cmd/fnv/RCS/hash_32.c,v $
*
***
*
* Fowler/Noll/Vo hash
*
* The basis of this hash algorithm was taken from an idea sent
* as reviewer comments to the IEEE POSIX P1003.2 committee by:
*
* Phong Vo (http://www.research.att.com/info/kpv/)
* Glenn Fowler (http://www.research.att.com/~gsf/)
*
* In a subsequent ballot round:
*
* Landon Curt Noll (http://www.isthe.com/chongo/)
*
* improved on their algorithm. Some people tried this hash
* and found that it worked rather well. In an EMail message
* to Landon, they named it the ``Fowler/Noll/Vo'' or FNV hash.
*
* FNV hashes are designed to be fast while maintaining a low
* collision rate. The FNV speed allows one to quickly hash lots
* of data while maintaining a reasonable collision rate. See:
*
* http://www.isthe.com/chongo/tech/comp/fnv/index.html
*
* for more details as well as other forms of the FNV hash.
***
*
* NOTE: The FNV-0 historic hash is not recommended. One should use
* the FNV-1 hash instead.
*
* To use the 32 bit FNV-0 historic hash, pass FNV0_32_INIT as the
* Fnv32_t hashval argument to fnv_32_buf() or fnv_32_str().
*
* To use the recommended 32 bit FNV-1 hash, pass FNV1_32_INIT as the
* Fnv32_t hashval argument to fnv_32_buf() or fnv_32_str().
*
***
*
* Please do not copyright this code. This code is in the public domain.
*
* LANDON CURT NOLL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
* INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO
* EVENT SHALL LANDON CURT NOLL BE LIABLE FOR ANY SPECIAL, INDIRECT OR
* CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
* USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
* OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
* PERFORMANCE OF THIS SOFTWARE.
*
* By:
* chongo <Landon Curt Noll> /\oo/\
* http://www.isthe.com/chongo/
*
* Share and Enjoy! :-)
*/
#include <stdlib.h>
#include "fnv.h"
/*
* 32 bit magic FNV-0 and FNV-1 prime
*/
#define FNV_32_PRIME ((Fnv32_t)0x01000193)
/*
* fnv_32_buf - perform a 32 bit Fowler/Noll/Vo hash on a buffer
*
* input:
* buf - start of buffer to hash
* len - length of buffer in octets
* hval - previous hash value or 0 if first call
*
* returns:
* 32 bit hash as a static hash type
*
* NOTE: To use the 32 bit FNV-0 historic hash, use FNV0_32_INIT as the hval
* argument on the first call to either fnv_32_buf() or fnv_32_str().
*
* NOTE: To use the recommended 32 bit FNV-1 hash, use FNV1_32_INIT as the hval
* argument on the first call to either fnv_32_buf() or fnv_32_str().
*/
Fnv32_t
fnv_32_buf(void *buf, size_t len, Fnv32_t hval)
{
unsigned char *bp = (unsigned char *)buf; /* start of buffer */
unsigned char *be = bp + len; /* beyond end of buffer */
/*
* FNV-1 hash each octet in the buffer
*/
while (bp < be) {
/* multiply by the 32 bit FNV magic prime mod 2^32 */
#if defined(NO_FNV_GCC_OPTIMIZATION)
hval *= FNV_32_PRIME;
#else
hval += (hval<<1) + (hval<<4) + (hval<<7) + (hval<<8) + (hval<<24);
#endif
/* xor the bottom with the current octet */
hval ^= (Fnv32_t)*bp++;
}
/* return our new hash value */
return hval;
}
/*
* fnv_32_str - perform a 32 bit Fowler/Noll/Vo hash on a string
*
* input:
* str - string to hash
* hval - previous hash value or 0 if first call
*
* returns:
* 32 bit hash as a static hash type
*
* NOTE: To use the 32 bit FNV-0 historic hash, use FNV0_32_INIT as the hval
* argument on the first call to either fnv_32_buf() or fnv_32_str().
*
* NOTE: To use the recommended 32 bit FNV-1 hash, use FNV1_32_INIT as the hval
* argument on the first call to either fnv_32_buf() or fnv_32_str().
*/
Fnv32_t
fnv_32_str(char *str, Fnv32_t hval)
{
unsigned char *s = (unsigned char *)str; /* unsigned string */
/*
* FNV-1 hash each octet in the buffer
*/
while (*s) {
/* multiply by the 32 bit FNV magic prime mod 2^32 */
#if defined(NO_FNV_GCC_OPTIMIZATION)
hval *= FNV_32_PRIME;
#else
hval += (hval<<1) + (hval<<4) + (hval<<7) + (hval<<8) + (hval<<24);
#endif
/* xor the bottom with the current octet */
hval ^= (Fnv32_t)*s++;
}
/* return our new hash value */
return hval;
}

18
longlong.h Executable file
View File

@ -0,0 +1,18 @@
/*
* DO NOT EDIT -- generated by the Makefile
*/
#if !defined(__LONGLONG_H__)
#define __LONGLONG_H__
/* do we have/want to use a long long type? */
#define HAVE_64BIT_LONG_LONG /* yes */
/*
* NO64BIT_LONG_LONG undef HAVE_64BIT_LONG_LONG
*/
#if defined(NO64BIT_LONG_LONG)
#undef HAVE_64BIT_LONG_LONG
#endif /* NO64BIT_LONG_LONG */
#endif /* !__LONGLONG_H__ */

361
roxanne_db.c
View File

@ -20,86 +20,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <errno.h>
#include <sys/socket.h>
#include <netdb.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/select.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/ipc.h>
#include <sys/shm.h>
#include <sys/mman.h>
#include <semaphore.h>
#include <math.h>
#include <stdbool.h>
#include "longlong.h"
#include "fnv.h"
/*
Here is the order for setting up a networked connection:
socket()
bind()
listen()
accept()
*/
// Stuff for .h
// Constants -n- Macros
#define BACKLOG 25
#define BLOCK_SIZE 4096
#define MAX_BLOCKS 134217728
#define BLOCK_BITMAP_BYTES 16777216
#define MSG_SIZE 65536
#define HASH_BITS 16
#define IDX_ENTRY_SIZE 1024
#define KEY_LEN (IDX_ENTRY_SIZE - 3*(sizeof(int)))
struct idx { // structure for an index record.
char key[KEY_LEN];
int block_offset; // starting block in the db file.
int length; // db blocks consumed.
int next; // overflow ptr to next index_record on disk.
};
struct db_ptr { // a structure that points to a value in the db file.
int block_offset;
int blocks;
};
// Function signatures
int start_listening(char* port, int backlog);
void sigchld_handler(int s);
void sigterm_handler_parent(int s);
void sigterm_handler_child(int s);
int get_hash_val(int bits, char* key);
int guts(int accept_fd, int listen_fd);
int extract_command(char* msg, int msglen);
int write_record(char* key, char* data);
int write_index(char* key, int block_offset, int length);
int parse_create(char msg[], int msglen, char** key, char** value);
int bit_array_set(char bit_array[], int bit);
int bit_array_test(char bit_array[], int bit);
int bit_array_clear(char bit_array[], int bit);
int find(char* key);
int create_block_reservation(int blocks_needed);
char* read_record(struct db_ptr db_rec);
void hash_write_lock(int hash_number);
void hash_write_unlock(int hash_number);
void cleanup_and_exit();
void usage(char *argv);
#include "roxanne_db.h"
// Globals
sem_t* DB_WRITE_LOCK;
@ -123,8 +44,8 @@ int main(int argc, char* argv[]) {
char idx_file[4096] = "/var/roxanne/idx";
char block_bitmap_file[4096] = "/var/roxanne/block_bitmap";
int chld;
int shm_block_offset_id;
key_t shm_block_offset_key = 1;
//int shm_block_offset_id;
//key_t shm_block_offset_key = 1;
int i;
int ch;
@ -202,7 +123,7 @@ int main(int argc, char* argv[]) {
}
if ((SHM_HASHBUCKET_BITMAP = mmap((caddr_t)0, ((1<<HASH_BITS)/8), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANON, -1, 0)) == MAP_FAILED) {
perror("Problem mmapping the hasj bitmap");
perror("Problem mmapping the hash bitmap");
exit(-1);
}
@ -337,16 +258,16 @@ int start_listening(char* port, int backlog) {
int extract_command(char* msg, int msglen) {
char* commands[4] = { "create: ", // 0
"read: ", // 1
"delete: "}; // 2
char* commands[3] = { "create ", // 0
"read ", // 1
"delete "}; // 2
int i = 0;
int cmdlen;
int max_chars = 0;
for (; i < 3; i++) {
cmdlen = strlen(commands[i]);
max_chars = msglen < cmdlen ? msglen : cmdlen;
if (strncmp(commands[i], msg, max_chars) == 0) return(i);
if (memcmp(commands[i], msg, max_chars) == 0) return(i);
}
return(-1);
@ -402,11 +323,11 @@ int create_block_reservation(int blocks_needed) {
struct db_ptr find_db_ptr(char* key) {
// returns an offset in the index for the given key.
int hash_id = get_hash_val(HASH_BITS, key);
struct idx index_rec = {};
struct db_ptr db_rec = {.block_offset = -1, .blocks = -1};
int result;
int pos = hash_id * IDX_ENTRY_SIZE;
int hash_id = get_hash_val(HASH_BITS, key);
struct idx index_rec = {};
struct db_ptr db_rec = {.block_offset = -1, .blocks = -1};
int result;
int64_t pos = hash_id * IDX_ENTRY_SIZE;
while (1) {
@ -418,11 +339,11 @@ struct db_ptr find_db_ptr(char* key) {
}
if (result < IDX_ENTRY_SIZE) { // Somehow the read failed.
perror("index read failed in function find");
perror("index read failed in function find_db_ptr");
return db_rec;
}
if ((strncmp(key, index_rec.key, KEY_LEN - 1)) == 0) {// found a match
if ((memcmp(key, index_rec.key, KEY_LEN)) == 0) {// found a match
db_rec.block_offset = index_rec.block_offset;
db_rec.blocks = index_rec.length;
return db_rec;
@ -436,10 +357,10 @@ struct db_ptr find_db_ptr(char* key) {
int find(char* key) {
// returns an offset in the index for the given key.
int hash_id = get_hash_val(HASH_BITS, key);
struct idx index_rec = {};
int result;
int pos = hash_id * IDX_ENTRY_SIZE;
int hash_id = get_hash_val(HASH_BITS, key);
struct idx index_rec = {};
int result;
int64_t pos = hash_id * IDX_ENTRY_SIZE;
while (1) {
@ -455,7 +376,7 @@ int find(char* key) {
return -1;
}
if ((strncmp(key, index_rec.key, KEY_LEN - 1)) == 0) return pos; // found
if ((memcmp(key, index_rec.key, KEY_LEN)) == 0) return pos; // found
if ((pos = index_rec.next) == 0) return -2; // no next record.
@ -466,12 +387,12 @@ int find(char* key) {
int write_index(char* key, int block_offset, int length) {
int hash_id = get_hash_val(HASH_BITS, key);
struct idx index_rec = {};
struct idx* index_rec_ptr;
int result;
int pos = hash_id * IDX_ENTRY_SIZE;
int find_results = find(key);
int hash_id = get_hash_val(HASH_BITS, key);
struct idx index_rec = {};
struct idx* index_rec_ptr;
int result;
int64_t pos = hash_id * IDX_ENTRY_SIZE;
int find_results = find(key);
index_rec_ptr = &index_rec;
@ -525,7 +446,6 @@ int write_index(char* key, int block_offset, int length) {
index_rec.next = lseek(IDX_FD, 0, SEEK_END);
pwrite(IDX_FD, (void*)index_rec_ptr, IDX_ENTRY_SIZE, pos); // update current rec with pointer to next.
pos = index_rec.next;
fprintf(stderr, "pos is %d\n", pos);
index_rec.next = 0;
index_rec.block_offset = block_offset;
index_rec.length = length;
@ -544,10 +464,10 @@ int write_index(char* key, int block_offset, int length) {
char* read_record(struct db_ptr db_rec) {
// read a record from the db file at the given offset.
char* buffer;
int byte_count = db_rec.blocks * BLOCK_SIZE;
int byte_offset = db_rec.block_offset * BLOCK_SIZE;
int bytes_read = 0;
char* buffer;
int byte_count = db_rec.blocks * BLOCK_SIZE;
int64_t byte_offset = db_rec.block_offset * BLOCK_SIZE;
int bytes_read = 0;
// Make a temporary buffer that is at least as big as the
// data payload that we need to read. The buffer will be
@ -574,8 +494,8 @@ int delete_record(char* key) {
void* buffer;
int byte_count = 0;
int block_offset;
int byte_offset;
int pos = 0;
int64_t byte_offset;
int64_t pos = 0;
int result;
struct idx index_rec;
int hash_id = get_hash_val(HASH_BITS, key);
@ -585,19 +505,27 @@ int delete_record(char* key) {
hash_write_lock(hash_id);
pos = find(key);
if (pos <= 0) {
fprintf(stderr, "Call to find() failed with %d.\n", pos);
if (pos == -1) {
fprintf(stderr, "Call to find() failed with %lld.\n", pos);
hash_write_unlock(hash_id);
return(-1);
}
if (pos == -2) {
hash_write_unlock(hash_id);
return(-1); // Not in the index
}
// Fetch the index record so we can find the blocks to delete
result = pread(IDX_FD, (void*)&index_rec, IDX_ENTRY_SIZE, pos);
if (result == 0) {
fprintf(stderr, "EOF encoutered unexpectedly.\n");
hash_write_unlock(hash_id);
return(-1);
}
if (result < IDX_ENTRY_SIZE) { // Somehow the read failed.
perror("index read failed in function delete_record");
hash_write_unlock(hash_id);
return(-1);
}
@ -639,7 +567,7 @@ int write_record(char* key, char* value) {
void* buffer;
int byte_count = 0;
int block_offset;
int byte_offset;
int64_t byte_offset;
int index_result;
int hash_id = get_hash_val(HASH_BITS, key);
@ -701,18 +629,20 @@ int write_record(char* key, char* value) {
int guts(int accept_fd, int listen_fd) {
char buffer[512] = ""; // recv buffer
char msg[MSG_SIZE] = ""; // Holds our incoming and outgoing messages.
char buffer[512] = ""; // recv buffer
char msg[MSG_SIZE] = ""; // Holds our incoming and outgoing messages.
void *msg_cursor;
char response[MSG_SIZE] = "";
int msglen = 0; // length of the assembled message that we receive.
int recvlen = 0; // how many bytes recv call returns.
int responselen = 0;
int length = 0;
char* key;
char* value;
char* cmd_offset;
struct db_ptr db_rec;
char response[MSG_SIZE] = "";
int msglen = 0; // length of the assembled message that we receive.
int recvlen = 0; // how many bytes recv call returns.
int responselen = 0;
//int length = 0;
//char key[KEY_LEN];
//char* value;
//char* part;
//char* previous_part;
//char* cmd_offset;
//struct db_ptr db_rec;
int retval;
@ -756,68 +686,28 @@ int guts(int accept_fd, int listen_fd) {
memcpy(msg_cursor, (void*)buffer, recvlen);
msg_cursor += recvlen;
if (memchr((void*)buffer, '\0', recvlen)) break; // Got a terminator character. Go process our message.
if (memchr((void*)buffer, '\r', recvlen)) break; // Got a terminator character. Go process our message.
if (memchr((void*)buffer, '\n', recvlen)) break; // Got a terminator character. Go process our message.
}
//length = 0;
//part = NULL;
//previous_part = NULL;
//key[0] = '\0';
switch (extract_command(msg, msglen)) {
case 0: // create
cmd_offset = &(msg[8]); // move beyond c-r-e-a-t-e-:-[space]
key = strsep(&cmd_offset, "&");
if ((cmd_offset == NULL) || (*key == '\0')) {
sprintf(response, "Failed to extract key.\n");
break;
}
if ((length = strnlen(key, KEY_LEN)) >= KEY_LEN) {
sprintf(response, "Key exceeds %lu bytes.\n", KEY_LEN);
break;
}
value = key + length + 1;
if (*value == '\0') {
sprintf(response, "No value supplied\n");
break;
}
retval = write_record(key, value);
if (retval == 0) {
sprintf(response, "Write OK.\n", key, value);
} else if (retval == -2) { // key already exists.
sprintf(response, "Write failed. Key exists in the index.\n", key, value);
} else {
sprintf(response, "write_record() failed. Don't know why. key = %s value = %s\n", key, value);
}
create_command(msg, response);
break;
case 1: // read
key = &(msg[6]); // move beyond r-e-a-d-:[space]
if (*key == '\0') {
sprintf(response, "No key supplied\n");
break;
}
db_rec = find_db_ptr(key);
if (db_rec.block_offset != -1) {
value = read_record(db_rec);
sprintf(response, "%s", value);
free(value);
} else {
sprintf(response, "Not found.\n");
}
read_command(msg, response);
break;
case 2:
sprintf(response, "Got a delete command.\n");
key = &(msg[8]); // move beyond d-e-l-e-t-e-:[space]
if (*key == '\0') {
sprintf(response, "No key supplied\n");
break;
}
if (delete_record(key) == 0) {
sprintf(response, "Delete OK.\n");
} else {
sprintf(response, "Delete failed.\n");
}
case 2: // delete
delete_command(msg, response);
break;
default:
@ -894,6 +784,117 @@ void hash_write_unlock(int hash_number) {
}
void create_command(char msg[], char response[]) {
int length = 0;
char* part = NULL;
char* previous_part = NULL;
int retval = 0;
char key[KEY_LEN] = "";
if ((part = strtok(msg, "/")) == NULL) {
sprintf(response, "Missing key.\n");
return;
}
for (part = strtok(NULL, "\r\n/"); part; part = strtok(NULL, "\r\n/")) {
if (previous_part != NULL) {
length += strlen(previous_part);
if (length > KEY_LEN - 1) {
sprintf(response, "Key too large.\n");
return;
}
strcat(key, previous_part);
}
previous_part = part;
}
if (key[0] == '\0') {
sprintf(response, "Failed to get value.\n");
return;
}
retval = write_record(key, previous_part);
if (retval == 0) {
sprintf(response, "Write OK.\n");
} else if (retval == -2) { // key already exists.
sprintf(response, "Write failed. Key exists in the index.\n");
} else {
sprintf(response, "write_record() failed. Don't know why.\n");
}
}
void read_command(char msg[], char response[]) {
char* part = NULL;
char* value;
int retval = 0;
char key[KEY_LEN] = "";
int length = 0;
struct db_ptr db_rec;
if ((part = strtok(msg, "/")) == NULL) {
sprintf(response, "Missing key.\n");
return;
}
for (part = strtok(NULL, "\r\n/"); part; part = strtok(NULL, "\r\n/")) {
length += strlen(part);
if (length > KEY_LEN - 1) {
sprintf(response, "Key too long.\n");
part = NULL;
break;
}
strcat(key, part);
}
if (length == 0) {
sprintf(response, "Failed to extract key.\n");
return;
}
db_rec = find_db_ptr(key);
if (db_rec.block_offset != -1) {
value = read_record(db_rec);
sprintf(response, "%s\n", value);
free(value);
} else {
sprintf(response, "Not found.\n");
}
}
void delete_command(char msg[], char response[]) {
char* part = NULL;
char key[KEY_LEN] = "";
int length = 0;
if ((part = strtok(msg, "/")) == NULL) {
sprintf(response, "Missing key.\n");
return;
}
for (part = strtok(NULL, "\r\n/"); part; part = strtok(NULL, "\r\n/")) {
length += strlen(part);
if (length > KEY_LEN - 1) {
sprintf(response, "Key too long.\n");
part = NULL;
break;
}
strcat(key, part);
}
if (length == 0) {
sprintf(response, "Failed to extract key.\n");
return;
}
if (delete_record(key) == 0) {
sprintf(response, "Delete OK.\n");
} else {
sprintf(response, "Delete failed.\n");
}
}
void usage(char *argv) {
fprintf(stderr, "usage: %s [-h listen_addr] [-p listen_port] [-d /path/to/db/directory]\n", argv);

107
roxanne_db.h Normal file
View File

@ -0,0 +1,107 @@
/*
Copyright (c) 2011 Joseph Rothrock (rothrock@rothrock.org)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <errno.h>
#include <sys/socket.h>
#include <netdb.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/select.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/ipc.h>
#include <sys/shm.h>
#include <sys/mman.h>
#include <semaphore.h>
#include <signal.h>
#include <math.h>
#include <stdint.h>
#include <stdbool.h>
#include <libgen.h>
#include "longlong.h"
#include "fnv.h"
/*
Here is the order for setting up a networked connection:
socket()
bind()
listen()
accept()
*/
// Stuff for .h
// Constants -n- Macros
#define BACKLOG 25
#define BLOCK_SIZE 4096
#define MAX_BLOCKS 1073741824
#define BLOCK_BITMAP_BYTES 134217728
#define MSG_SIZE 65536
#define HASH_BITS 16
#define IDX_ENTRY_SIZE 1024
#define KEY_LEN (IDX_ENTRY_SIZE - 3*(sizeof(int)))
struct idx { // structure for an index record.
char key[KEY_LEN];
int block_offset; // starting block in the db file.
int length; // db blocks consumed.
int next; // overflow ptr to next index_record on disk.
};
struct db_ptr { // a structure that points to a value in the db file.
int block_offset;
int blocks;
};
// Function signatures
int start_listening(char* port, int backlog);
void sigchld_handler(int s);
void sigterm_handler_parent(int s);
void sigterm_handler_child(int s);
int get_hash_val(int bits, char* key);
int guts(int accept_fd, int listen_fd);
int extract_command(char* msg, int msglen);
int write_record(char* key, char* data);
int write_index(char* key, int block_offset, int length);
int parse_create(char msg[], int msglen, char** key, char** value);
int bit_array_set(char bit_array[], int bit);
int bit_array_test(char bit_array[], int bit);
int bit_array_clear(char bit_array[], int bit);
int find(char* key);
int create_block_reservation(int blocks_needed);
char* read_record(struct db_ptr db_rec);
void hash_write_lock(int hash_number);
void hash_write_unlock(int hash_number);
void cleanup_and_exit();
void usage(char *argv);
void create_command(char msg[], char response[]);
void read_command(char msg[], char response[]);
void delete_command(char msg[], char response[]);

2
test_data Normal file
View File

File diff suppressed because one or more lines are too long