Merge pull request #8 from SuperconductZB/guangzheliu/layer1dev

Guangzheliu/layer1dev
This commit is contained in:
FactorialN 2023-11-12 14:40:42 -08:00 committed by GitHub
commit eb8eebc099
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 603 additions and 9 deletions

470
include/fs.h Normal file
View File

@ -0,0 +1,470 @@
#include "rawdisk.h"
/*****************************************************
30GB Disk low-level operation and data structure: spuerblock, inode, and buffer cache
512 bytes sector for 1 block, 62914560 block(sector)
4K bytes sector for 1 block, 7864320 block(sector)
one inode equipped with one 512 bytes block
*****************************************************/
#define SECTOR_SIZE 512
#define IO_BLOCK_SIZE 4096
#define MAX_INODE 524288
#define MAX_BLOCKNUM MAX_INODE*2 //62914560
class SuperBlock{
public:
SuperBlock(){
}
~SuperBlock(){
}
static u_int64_t getFreeListHead(RawDisk &disk){
char buffer[8] = {0};
disk.rawdisk_read(0, buffer, sizeof(buffer));
u_int64_t t = 0;
for (int j = 0; j < 8; j++)
t = t | (((u_int64_t)(unsigned char)buffer[j])<<(8*j));
return t;
}
static void writeFreeListHead(RawDisk &disk, u_int64_t t){
char buffer[8] = {0};
for (int j = 0; j < 8; j++){
buffer[j] = (t >> (8 * j)) & 0xFF;
}
disk.rawdisk_write(0, buffer, sizeof(buffer));
}
static u_int64_t getFreeINodeHead(RawDisk &disk){
char buffer[8] = {0};
disk.rawdisk_read(8, buffer, sizeof(buffer));
u_int64_t t = 0;
for (int j = 0; j < 8; j++)
t = t | (((u_int64_t)(unsigned char)buffer[j])<<(8*j));
return t;
}
static void writeFreeINodeHead(RawDisk &disk, u_int64_t t){
char buffer[8] = {0};
for (int j = 0; j < 8; j++){
buffer[j] = (t >> (8 * j)) & 0xFF;
}
disk.rawdisk_write(8, buffer, sizeof(buffer));
}
};
class INode{
// direct datablocks
u_int64_t blocks[48];
// indirect address
u_int64_t single_indirect, double_indirect, triple_indirect;
// other
u_int64_t uid;
u_int64_t gid;
u_int64_t permissions;
u_int64_t size;
u_int64_t block_number;
public:
void read_get_byte(u_int64_t &t, int &current_pos, char *buffer){
t = 0;
for (int j = 0; j < 8; j++)
t = t | (((u_int64_t)(unsigned char)buffer[j+current_pos])<<(8*j));
current_pos += 8;
}
static u_int64_t read_byte_at(int current_pos, char *buffer){
u_int64_t t = 0;
for (int j = 0; j < 8; j++)
t = t | (((u_int64_t)(unsigned char)buffer[j+current_pos])<<(8*j));
return t;
}
void inode_construct(u_int64_t blockNumber, RawDisk &disk){
char buffer[SECTOR_SIZE] = {0};
disk.rawdisk_read(blockNumber*SECTOR_SIZE, buffer, sizeof(buffer));
block_number = blockNumber;
int current_pos = 0;
// initialize blocks
for (int i = 0; i < 48; i++){
read_get_byte(blocks[i], current_pos, buffer);
}
read_get_byte(single_indirect, current_pos, buffer);
read_get_byte(double_indirect, current_pos, buffer);
read_get_byte(triple_indirect, current_pos, buffer);
read_get_byte(uid, current_pos, buffer);
read_get_byte(gid, current_pos, buffer);
read_get_byte(permissions, current_pos, buffer);
read_get_byte(size, current_pos, buffer);
}
void write_get_byte(u_int64_t t, int &current_pos, char *buffer){
for (int j = 0; j < 8; j++){
buffer[j+current_pos] = t & (((u_int64_t)1<<(8))-1);
t >>= 8;
}
current_pos += 8;
}
static void write_byte_at(u_int64_t t, int current_pos, char *buffer){
for (int j = 0; j < 8; j++){
buffer[j+current_pos] = t & (((u_int64_t)1<<(8))-1);
t >>= 8;
}
}
void inode_save(RawDisk &disk){
char buffer[SECTOR_SIZE] = {0};
int current_pos = 0;
for (int i = 0; i < 48; i++){
write_get_byte(blocks[i], current_pos, buffer);
}
write_get_byte(single_indirect, current_pos, buffer);
write_get_byte(double_indirect, current_pos, buffer);
write_get_byte(triple_indirect, current_pos, buffer);
write_get_byte(uid, current_pos, buffer);
write_get_byte(gid, current_pos, buffer);
write_get_byte(permissions, current_pos, buffer);
write_get_byte(size, current_pos, buffer);
disk.rawdisk_write(block_number*SECTOR_SIZE, buffer, sizeof(buffer));
}
u_int64_t datablock_allocate_in_list(RawDisk &disk){
//find a free data block
u_int64_t freeListHead = SuperBlock::getFreeListHead(disk);
/*
1. initialization
2. data block starting position
3. r/w between storage and rawdisk to maintain
*/
char buffer[IO_BLOCK_SIZE] = {0};
u_int64_t freeBlockNum = 0;
disk.rawdisk_read(freeListHead, buffer, sizeof(buffer));
for (int i = 8; i < 264; i++){
if((i < 263 && buffer[i] != -1) || (i == 263 && buffer[i] != 127)){
int j = 0;
for (j = 0; j < 8; j++){
if ((buffer[i]&(1<<j)) == 0){
buffer[i] |= (1<<j);
break;
}
}
if (j < 8){
freeBlockNum = freeListHead + ((i-8)*8 + j + 1)*IO_BLOCK_SIZE;
break;
}
}
}
disk.rawdisk_write(freeListHead, buffer, sizeof(buffer));
bool notFull = false;
for (int i = 8; i < 264; i++){
if((i < 263 && buffer[i] != -1) || (i == 263 && buffer[i] != 127)){
notFull = true;
break;
}
}
if (!notFull){
printf("HEADER REMOVAL DETECTED %llu %llu\n", freeListHead, freeBlockNum);
u_int64_t next_header = read_byte_at(0, buffer);
SuperBlock::writeFreeListHead(disk, next_header);
}
return freeBlockNum;
}
bool allo_single_indirect(RawDisk &disk, u_int64_t &single_i, u_int64_t freeBlockNum) {
if (single_i == 0){
single_i = datablock_allocate_in_list(disk);
char new_buffer[IO_BLOCK_SIZE] = {0};
disk.rawdisk_write(single_i, new_buffer, sizeof(new_buffer));
}
bool inSingle = false;
char buffer[IO_BLOCK_SIZE] = {0};
disk.rawdisk_read(single_i, buffer, sizeof(buffer));
for (int i = 0; i < IO_BLOCK_SIZE; i+=8){
u_int64_t addr = read_byte_at(i, buffer);
if(addr == 0){
inSingle = true;
write_byte_at(freeBlockNum, i, buffer);
disk.rawdisk_write(single_i, buffer, sizeof(buffer));
break;
}
}
return inSingle;
}
bool allo_double_indirect(RawDisk &disk, u_int64_t &double_i, u_int64_t freeBlockNum) {
if (double_i == 0){
double_i = datablock_allocate_in_list(disk);
char new_buffer[IO_BLOCK_SIZE] = {0};
disk.rawdisk_write(double_i, new_buffer, sizeof(new_buffer));
}
bool inDouble = false;
char buffer[IO_BLOCK_SIZE] = {0};
disk.rawdisk_read(double_i, buffer, sizeof(buffer));
for (int i = 0; i < IO_BLOCK_SIZE; i+=8){
u_int64_t addr = read_byte_at(i, buffer);
bool flag = allo_single_indirect(disk, addr, freeBlockNum);
if (flag){
write_byte_at(addr, i, buffer);
disk.rawdisk_write(double_i, buffer, sizeof(buffer));
inDouble = true;
break;
}
}
return inDouble;
}
bool allo_triple_indirect(RawDisk &disk, u_int64_t &triple_i, u_int64_t freeBlockNum) {
if (triple_i == 0){
triple_i = datablock_allocate_in_list(disk);
char new_buffer[IO_BLOCK_SIZE] = {0};
disk.rawdisk_write(triple_i, new_buffer, sizeof(new_buffer));
}
bool inTriple = false;
char buffer[IO_BLOCK_SIZE] = {0};
disk.rawdisk_read(triple_i, buffer, sizeof(buffer));
for (int i = 0; i < IO_BLOCK_SIZE; i+=8){
u_int64_t addr = read_byte_at(i, buffer);
bool flag = allo_double_indirect(disk, addr, freeBlockNum);
if (flag){
write_byte_at(addr, i, buffer);
disk.rawdisk_write(triple_i, buffer, sizeof(buffer));
inTriple = true;
break;
}
}
return inTriple;
}
// allowcate 1 datablock and add to the end of the file
u_int64_t datablock_allocate(RawDisk &disk){
//do we need to check dynamic?
//add the data block to blocks, single, double, triple
u_int64_t freeBlockNum = datablock_allocate_in_list(disk);
bool inBlocks = false;
for (int i = 0; i < 48; i++)
if(blocks[i] == 0){
inBlocks = true;
blocks[i] = freeBlockNum;
break;
}
if(!inBlocks){
bool inSingle = allo_single_indirect(disk, single_indirect, freeBlockNum);
if (!inSingle){
bool inDouble = allo_double_indirect(disk, double_indirect, freeBlockNum);
if (!inDouble){
bool inTriple = allo_triple_indirect(disk, triple_indirect, freeBlockNum);
// wait to deal with too big files
}
}
}
//return the block number
inode_save(disk);
return freeBlockNum;
}
void datablock_deallocate_in_list(u_int64_t freeBlockNum, RawDisk &disk) {
// find the related 2048block head
u_int64_t freeBlockHead = ((freeBlockNum/SECTOR_SIZE-MAX_INODE)/(8*2048)*(8*2048)+MAX_INODE)*SECTOR_SIZE;
// mark it alive in its bitmap
char buffer[IO_BLOCK_SIZE] = {0};
bool nowInList = false;
disk.rawdisk_read(freeBlockHead, buffer, sizeof(buffer));
for (int i = 8; i < 264; i++){
if((i < 263 && buffer[i] != -1) || (i == 263 && buffer[i] != 127)){
nowInList = true;
}
}
u_int64_t inBlockPos = (freeBlockNum-freeBlockHead)/IO_BLOCK_SIZE-1;
buffer[8+inBlockPos/8] &= (-1)^(1<<(inBlockPos%8));
// if its bitmap was 0, add it back to the list head
if(!nowInList){
u_int64_t freeListHead = SuperBlock::getFreeListHead(disk);
write_byte_at(freeListHead, 0, buffer);
SuperBlock::writeFreeListHead(disk, freeBlockHead);
}
disk.rawdisk_write(freeBlockHead, buffer, sizeof(buffer));
}
u_int64_t deallo_single_indirect(RawDisk &disk, u_int64_t &single_i){
if (single_i == 0){
return 0;
}
u_int64_t freeBlockNum = 0;
char buffer[IO_BLOCK_SIZE] = {0};
int delpoint = -1;
disk.rawdisk_read(single_i, buffer, sizeof(buffer));
for (int i=4088; i >= 0; i-=8){
u_int64_t addr = read_byte_at(i, buffer);
if(addr != 0){
freeBlockNum = addr;
addr = 0;
write_byte_at(addr, i, buffer);
delpoint = i;
break;
}
}
disk.rawdisk_write(single_i, buffer, sizeof(buffer));
u_int64_t addr = read_byte_at(0, buffer);
if (delpoint == 0 && addr == 0){
datablock_deallocate_in_list(single_i, disk);
single_i = 0;
}
return freeBlockNum;
}
u_int64_t deallo_double_indirect(RawDisk &disk, u_int64_t &double_i){
if (double_i == 0){
return false;
}
u_int64_t freeBlockNum = 0;
char buffer[IO_BLOCK_SIZE] = {0};
int delpoint = -1;
disk.rawdisk_read(double_i, buffer, sizeof(buffer));
for (int i=4088; i >= 0; i-=8){
u_int64_t addr = read_byte_at(i, buffer);
u_int64_t inSingle = deallo_single_indirect(disk, addr);
if (inSingle){
freeBlockNum = inSingle;
write_byte_at(addr, i, buffer);
delpoint = i;
break;
}
}
disk.rawdisk_write(double_i, buffer, sizeof(buffer));
u_int64_t addr = read_byte_at(0, buffer);
if (delpoint == 0 && addr == 0){
datablock_deallocate_in_list(double_i, disk);
double_i = 0;
}
return freeBlockNum;
}
u_int64_t deallo_triple_indirect(RawDisk &disk, u_int64_t &triple_i){
if (triple_i == 0){
return false;
}
u_int64_t freeBlockNum = 0;
char buffer[IO_BLOCK_SIZE] = {0};
int delpoint = -1;
disk.rawdisk_read(triple_i, buffer, sizeof(buffer));
for (int i=4088; i >= 0; i-=8){
u_int64_t addr = read_byte_at(i, buffer);
u_int64_t inDouble = deallo_double_indirect(disk, addr);
if (inDouble){
freeBlockNum = inDouble;
write_byte_at(addr, i, buffer);
delpoint = i;
break;
}
}
disk.rawdisk_write(triple_i, buffer, sizeof(buffer));
u_int64_t addr = read_byte_at(0, buffer);
if (delpoint == 0 && addr == 0){
datablock_deallocate_in_list(triple_i, disk);
triple_i = 0;
}
return freeBlockNum;
}
// deallocate 1 datablock from the end of the file
u_int64_t datablock_deallocate(RawDisk &disk){
// find the last datablock and remove it from inode (triple->direct)
u_int64_t freeBlockNum = 0;
freeBlockNum = deallo_triple_indirect(disk, triple_indirect);
if(!freeBlockNum){
freeBlockNum = deallo_double_indirect(disk, double_indirect);
if(!freeBlockNum){
freeBlockNum = deallo_single_indirect(disk, single_indirect);
if(!freeBlockNum){
for(int i = 47; i>=0; i--)
if(blocks[i] != 0){
freeBlockNum = blocks[i];
blocks[i] = 0;
break;
}
// deal with empty
}
}
}
// add it back to freeBlocklist
datablock_deallocate_in_list(freeBlockNum, disk);
inode_save(disk);
return freeBlockNum;
}
};
class INodeOperation{
// free list head is at super block (0): first 8 bytes
public:
//initialization of the rawdisk
void initialize(RawDisk &disk){
// initialize Inode list head
SuperBlock::writeFreeINodeHead(disk, 1);
for (u_int64_t i = 1; i < MAX_INODE; i++){
char buffer[SECTOR_SIZE] = {0};
u_int64_t t = i + 1;
if (t < MAX_INODE){
for (int j = 0; j < 8; j++){
buffer[j] = (t >> (8 * j)) & 0xFF;
}
}
disk.rawdisk_write(i*SECTOR_SIZE, buffer, sizeof(buffer));
}
SuperBlock::writeFreeListHead(disk, MAX_INODE*SECTOR_SIZE); // maximum inode number 2^19 0x80000
//Have tested this initialize function but MAX_BLOCK too much, MAX_INODE*2 works
for (u_int64_t i = MAX_INODE; i < MAX_BLOCKNUM-4096; i += 2048*8){
char buffer[IO_BLOCK_SIZE] = {0};
u_int64_t t = (i + 2048*8)*SECTOR_SIZE;
//t is address, storing in to buffer
for (int j = 0; j < 8; j++){
buffer[j] = (t >> (8 * j)) & 0xFF;
}
disk.rawdisk_write(i*SECTOR_SIZE, buffer, sizeof(buffer));
}
}
// allocate an inode from free inode list head and return the number of the inode
// the i-th inode is in the i-th block
u_int64_t inode_allocate(RawDisk &disk){
u_int64_t freeINodeHead = SuperBlock::getFreeINodeHead(disk);
char buffer[SECTOR_SIZE] = {0};
disk.rawdisk_read(freeINodeHead*SECTOR_SIZE, buffer, sizeof(buffer));
u_int64_t newINodeHead = INode::read_byte_at(0, buffer);
// deal with no more INode
SuperBlock::writeFreeINodeHead(disk, newINodeHead);
//to do: initialize the INode on disk at freeINodeHead
//return inode number
return freeINodeHead;
}
// free the inode and add it to the free inode list head
void inode_free(RawDisk &disk, u_int64_t INodeNumber){
u_int64_t freeINodeHead = SuperBlock::getFreeINodeHead(disk);
char buffer[SECTOR_SIZE] = {0};
INode::write_byte_at(freeINodeHead, 0, buffer);
disk.rawdisk_write(INodeNumber*SECTOR_SIZE, buffer, sizeof(buffer));
SuperBlock::writeFreeINodeHead(disk, INodeNumber);
}
//ignore for now
void inode_read(){
}
void inode_write(){
}
};

View File

@ -10,8 +10,8 @@ class RawDisk{
int fd;
const char* dir;
off_t numSectors;
off_t diskSize;
u_int64_t numSectors;
u_int64_t diskSize;
public:
RawDisk(const char *directory) : fd(-1), dir(nullptr), numSectors(0), diskSize(0) {
@ -48,8 +48,8 @@ public:
}
}
int rawdisk_read(off_t offset, char *buffer, size_t length) {
if (lseek(fd, offset, SEEK_SET) == (off_t)-1) {
int rawdisk_read(u_int64_t offset, char *buffer, size_t length) {
if (lseek(fd, offset, SEEK_SET) == (u_int64_t)-1) {
perror("Error seeking to offset");
return -1;
}
@ -64,8 +64,8 @@ public:
}
// Write a specified number of bytes at a given byte offset
int rawdisk_write(off_t offset, char *buffer, size_t length) {
if (lseek(fd, offset, SEEK_SET) == (off_t)-1) {
int rawdisk_write(u_int64_t offset, char *buffer, size_t length) {
if (lseek(fd, offset, SEEK_SET) == (u_int64_t)-1) {
perror("Error seeking to offset");
return -1;
}

View File

@ -1,4 +1,5 @@
set(TARGET_LAYER0 test_layer0)
set(TARGET_LAYER1_API test_layer1_API)
set(DIR_PLACE /dev/vdb)
# add test sources here ...
@ -6,6 +7,11 @@ add_executable(${TARGET_LAYER0}
# add need lib and source code here
layer0.cpp
)
add_executable(${TARGET_LAYER1_API}
# add need lib and source code here
layer1_API.cpp
)
# add test to activate ctest -VV
add_test(NAME ${TARGET_LAYER0} COMMAND sudo ./${TARGET_LAYER0} ${DIR_PLACE})
add_test(NAME ${TARGET_LAYER1_API} COMMAND sudo ./${TARGET_LAYER1_API} ${DIR_PLACE})

View File

@ -14,11 +14,11 @@ int main(int argc, char *argv[]) {
//printf("dir %s, numSectors %lld, diskSize %lld \n", H->dir, H->numSectors, H->diskSize);
//use number to substitute H->getnumSector(), getnumSectors() are not yest implemented
for(off_t i = 0; i < 10; i++) {
for(u_int64_t i = 0; i < 10; i++) {
H->rawdisk_write(i*512, buf, strlen(buf));//Change write_API
}
//use number to substitute H->getnumSector(), getnumSectors() are not yest implemented
for(off_t i = 0; i < 10; i++) {
for(u_int64_t i = 0; i < 10; i++) {
H->rawdisk_read(i*512, readBuffer, sizeof(readBuffer));//Change read_API
assert(strncmp(readBuffer, buf, strlen(buf)) == 0);
}

118
test/layer1_API.cpp Normal file
View File

@ -0,0 +1,118 @@
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include "fs.h"
#include <inttypes.h>
int main(int argc, char *argv[]) {
const char* d = (argc < 2) ? "/dev/vdc" : argv[1];
RawDisk *H = new RawDisk(d);
printf("test inode\n");
INodeOperation inop;
inop.initialize(*H);//for inode initialization and datablock initialization
char buffer[8] = {0};
/**************************test inode Initialization***************************/
//test the begining of inode 1th
H->rawdisk_read((1) * SECTOR_SIZE, buffer, sizeof(buffer));
u_int64_t t = 0;
for (int j = 0; j < 8; j++)
t |= ((u_int64_t)(unsigned char)buffer[j]) << (8 * j);
assert(t == 2);//the first 1th unused inode will store the next unused inode 2th
//test the number before end of inode 524286th
H->rawdisk_read((MAX_INODE - 2) * SECTOR_SIZE, buffer, sizeof(buffer));
t = 0;
for (int j = 0; j < 8; j++)
t |= ((u_int64_t)(unsigned char)buffer[j]) << (8 * j);
assert(t == MAX_INODE - 1);//store the maximun th inode
//test the end of inode 524287th
H->rawdisk_read((MAX_INODE - 1) * SECTOR_SIZE, buffer, sizeof(buffer));
t = 0;
for (int j = 0; j < 8; j++)
t |= ((u_int64_t)(unsigned char)buffer[j]) << (8 * j);
assert(t == 0);//the end of inode(524287th inode) do not have the next inode address
/**************************test datablock Initialization***************************/
//we separate 2048 4kB I/O block(1+2047) as a group and the first I/O block will manage the following 2047 I/O block usage.
//the first 8 bytes(0~7) in first I/O block store the address of next first I/O block, the following 256(8~263) bytes record 2047 I/O block usage.
//test the begining of free datablock
H->rawdisk_read((MAX_INODE) * SECTOR_SIZE, buffer, sizeof(buffer));//the begining of free datablock will store from address (MAX_INODE) * SECTOR_SIZE
t = 0;
for (int j = 0; j < 8; j++)
t |= ((u_int64_t)(unsigned char)buffer[j]) << (8 * j);
assert(t == (MAX_INODE+2048*8)*SECTOR_SIZE);//the first 8 bytes of 4k I/O block will store the next address(after 2048*4k I/O block)
//test the end of the datablock
H->rawdisk_read((MAX_BLOCKNUM - 2048*8) * SECTOR_SIZE, buffer, sizeof(buffer));
t = 0;
for (int j = 0; j < 8; j++)
t |= ((u_int64_t)(unsigned char)buffer[j]) << (8 * j);
assert(t == (MAX_BLOCKNUM)*SECTOR_SIZE);
/***************************test inode de/allocation**********************************/
//when requesting an inode, the inode_allocation will give you the inode number, we use inode_list to store the sequence allocate inode
//arrary version
int inode_list[20] = {0};
int record_free[10] = {0};//should do a pop up structure to record the free inode
int rec = 9;
//printf("Allocate 20 inode num:{");
for(int i=0;i<20;i++){
inode_list[i] = inop.inode_allocate(*H);
assert(inode_list[i] == i+1);
//printf(" %d", inode_list[i]);
}
//printf("}\n");
for(int i=10;i<20;i++){
inop.inode_free(*H,inode_list[i]);//free the 10 element from inode_list[10]
record_free[i-10] = inode_list[i];
}
//allocate again the last 10
printf("Allocate again: inode num:{");
for(int i=10;i<20;i++){
inode_list[i] = inop.inode_allocate(*H);
//printf("inode %d, rec_f %d\n,", inode_list[i],record_free[rec]);
assert(inode_list[i] == record_free[rec]);
rec--;
}
printf("}\n");
/***************************test direct blocks[48] de/allocation**********************************/
//after free the datablock, the program will find the first smallest address of datablock to give to the inode
//should test random resize each node, but should use datablock_free data structure to record
INode inode_inside[10];
u_int64_t rec_datablock_free[10][3] = {0};//array version
for(int i=0;i<10;i++){
inode_inside[i].inode_construct(inode_list[i],*H);
//printf("%dth data block starting addres: ", i);
for(int j=0;j<6;j++){
inode_inside[i].datablock_allocate(*H);
//printf("%d," ,inode_inside[i].datablock_allocate(*H));
}
//printf("\n");
}
for(int i=0;i<10;i++){
//printf("%dth data block free addres: ", i);
for(int j = 2;j >=0;j--){
rec_datablock_free[i][j] = inode_inside[i].datablock_deallocate(*H);
//printf("", rec_datablock_free[i][j]);
}
//printf("\n");
}
for(int i=0;i<10;i++){
//printf("%dth data block allocate again addres: ", i);
for(int j=0;j<3;j++){
assert(inode_inside[i].datablock_allocate(*H) == rec_datablock_free[i][j]);
//printf("%d," ,inode_inside[i].datablock_allocate(*H));
}
//printf("\n");
}
//printf("}\n");
delete H; // Delete the RawDisk object
return 0;
}