Skip to content

Commit

Permalink
Improve SC cache syncing.
Browse files Browse the repository at this point in the history
  • Loading branch information
@Piotrekol
Piotrekol committed Feb 13, 2018
1 parent c77d554 commit 4e8903c
Show file tree
Hide file tree
Showing 2 changed files with 74 additions and 23 deletions.
83 changes: 66 additions & 17 deletions osu!StreamCompanion/Code/Core/SqliteConnector.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ public class SqliteConnector : IDisposable, IMapDataStorer
{
readonly SQLiteConnection _mDbConnection;
private string DbFilename = "StreamCompanionCacheV2.db";
private int _schemaVersion = 1;
private SQLiteCommand _insertSql;
private SQLiteTransaction _transation;
public bool MassInsertIsActive => _transation != null;
Expand Down Expand Up @@ -51,6 +52,10 @@ public string GetTableDef()
}
public SqliteConnector()
{
_tableStruct.Fieldnames = new List<string>(new[] { "Raw", "TitleRoman", "ArtistRoman", "TitleUnicode", "ArtistUnicode", "Creator", "DiffName", "Mp3Name", "Md5", "OsuFileName", "MaxBpm", "MinBpm", "Tags", "State", "Circles", "Sliders", "Spinners", "EditDate", "ApproachRate", "CircleSize", "HpDrainRate", "OverallDifficulty", "SliderVelocity", "DrainingTime", "TotalTime", "PreviewTime", "MapId", "MapSetId", "ThreadId", "MapRating", "Offset", "StackLeniency", "Mode", "Source", "AudioOffset", "LetterBox", "Played", "LastPlayed", "IsOsz2", "Dir", "LastSync", "DisableHitsounds", "DisableSkin", "DisableSb", "BgDim", "Somestuff", "VideoDir", "StarsOsu" });
_tableStruct.Type = new List<string>(new[] { "VARCHAR", "VARCHAR", "VARCHAR", "VARCHAR", "VARCHAR", "VARCHAR", "VARCHAR", "VARCHAR", "VARCHAR", "VARCHAR", "DOUBLE", "DOUBLE", "VARCHAR", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "DATETIME", "DOUBLE", "DOUBLE", "DOUBLE", "DOUBLE", "DOUBLE", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "DOUBLE", "INTEGER", "VARCHAR", "INTEGER", "VARCHAR", "BOOL", "DATETIME", "BOOL", "VARCHAR", "DATETIME", "BOOL", "BOOL", "BOOL", "INTEGER", "INTEGER", "VARCHAR", "BLOB" });
_tableStruct.TypeModifiers = new List<string>(new[] { "NOT NULL", "NOT NULL", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL UNIQUE", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL", "NOT NULL", "NOT NULL", "NOT NULL ", "NOT NULL", "NOT NULL", "NOT NULL", "NOT NULL", "NOT NULL", "NOT NULL", "NOT NULL", "NOT NULL", "NOT NULL", "", "NOT NULL", "", "NOT NULL", "NOT NULL", "", "", "", "NOT NULL", "NOT NULL", "NOT NULL", "NOT NULL" });

CreateFile(DbFilename);
_mDbConnection = new SQLiteConnection("Data Source=" + DbFilename + ";Version=3;New=False;Compress=True;");
OpenConnection();
Expand All @@ -59,21 +64,65 @@ public SqliteConnector()

private void CreateTables()
{
_tableStruct.Fieldnames = new List<string>(new[] { "Raw", "TitleRoman", "ArtistRoman", "TitleUnicode", "ArtistUnicode", "Creator", "DiffName", "Mp3Name", "Md5", "OsuFileName", "MaxBpm", "MinBpm", "Tags", "State", "Circles", "Sliders", "Spinners", "EditDate", "ApproachRate", "CircleSize", "HpDrainRate", "OverallDifficulty", "SliderVelocity", "DrainingTime", "TotalTime", "PreviewTime", "MapId", "MapSetId", "ThreadId", "MapRating", "Offset", "StackLeniency", "Mode", "Source", "AudioOffset", "LetterBox", "Played", "LastPlayed", "IsOsz2", "Dir", "LastSync", "DisableHitsounds", "DisableSkin", "DisableSb", "BgDim", "Somestuff", "VideoDir", "StarsOsu" });
_tableStruct.Type = new List<string>(new[] { "VARCHAR", "VARCHAR", "VARCHAR", "VARCHAR", "VARCHAR", "VARCHAR", "VARCHAR", "VARCHAR", "VARCHAR", "VARCHAR", "DOUBLE", "DOUBLE", "VARCHAR", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "DATETIME", "DOUBLE", "DOUBLE", "DOUBLE", "DOUBLE", "DOUBLE", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "DOUBLE", "INTEGER", "VARCHAR", "INTEGER", "VARCHAR", "BOOL", "DATETIME", "BOOL", "VARCHAR", "DATETIME", "BOOL", "BOOL", "BOOL", "INTEGER", "INTEGER", "VARCHAR", "BLOB" });
_tableStruct.TypeModifiers = new List<string>(new[] { "NOT NULL", "NOT NULL", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL UNIQUE", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL ", "NOT NULL", "NOT NULL", "NOT NULL", "NOT NULL ", "NOT NULL", "NOT NULL", "NOT NULL", "NOT NULL", "NOT NULL", "NOT NULL", "NOT NULL", "NOT NULL", "NOT NULL", "", "NOT NULL", "", "NOT NULL", "NOT NULL", "", "", "", "NOT NULL", "NOT NULL", "NOT NULL","NOT NULL" });
string sql = "CREATE TABLE IF NOT EXISTS withID " + _tableStruct.GetTableDef();
NonQuery(sql);
sql = "CREATE TABLE IF NOT EXISTS withoutID " + _tableStruct.GetTableDef();
NonQuery(sql);

sql = "DROP TABLE IF EXISTS Temp";
NonQuery(sql);
sql = "CREATE TABLE IF NOT EXISTS Temp " + _tableStruct.GetTableDef();
NonQuery(sql);

sql = "CREATE TABLE IF NOT EXISTS `cfg` (`LastUpdateDate` TEXT)";
NonQuery(sql);
bool recteate = false;
var result = Query("SELECT name FROM sqlite_master WHERE type='table' AND name='cfg';");
if (result.HasRows)
{
result.Dispose();
//Check for old table struct with never got used(No data inside)
result = Query("SELECT * from cfg");
if (!result.HasRows)
{
//Old struct - drop everything
recteate = true;
}
else
{
//new struct - check schema version
result.Dispose();
result = Query("SELECT * from `cfg` where `Key` = 'SchemaVersion' ");
if (!result.HasRows || !result.Read())
recteate = true;
else
{
var version = Int32.Parse(result.GetString(1));
if (version != _schemaVersion)
recteate = true;
}
result.Dispose();


}
}
else
recteate = true;

if (recteate)
{
NonQuery("DROP TABLE IF EXISTS `cfg`;");
NonQuery("DROP TABLE IF EXISTS `Temp`;");
NonQuery("DROP TABLE IF EXISTS `withoutID`;");
NonQuery("DROP TABLE IF EXISTS `withID`;");


string sql = "CREATE TABLE IF NOT EXISTS withoutID " + _tableStruct.GetTableDef();
NonQuery(sql);

sql = "DROP TABLE IF EXISTS Temp";
NonQuery(sql);
sql = "CREATE TABLE IF NOT EXISTS Temp " + _tableStruct.GetTableDef();
NonQuery(sql);

var findIndex = _tableStruct.Fieldnames.FindIndex(s => s.Equals("MapId"));
_tableStruct.TypeModifiers[findIndex] = "NOT NULL UNIQUE";
sql = "CREATE TABLE IF NOT EXISTS withID " + _tableStruct.GetTableDef();
NonQuery(sql);

sql = "CREATE TABLE IF NOT EXISTS `cfg` (`Key` TEXT, `Value` TEXT)";
NonQuery(sql);
NonQuery($"insert into `cfg` values ('SchemaVersion','{_schemaVersion}')");
}

}


Expand Down Expand Up @@ -119,8 +168,8 @@ private void CreateFile(string filename)
}
catch (UnauthorizedAccessException ex)
{
throw new NonLoggableException(ex,"Could not save beatmap cache file due to insuffisient premissions"+
Environment.NewLine+"Please move this exectuable into a non-system folder");
throw new NonLoggableException(ex, "Could not save beatmap cache file due to insuffisient premissions" +
Environment.NewLine + "Please move this exectuable into a non-system folder");
}
}
private void OpenConnection()
Expand Down
14 changes: 8 additions & 6 deletions osu!StreamCompanion/Code/Core/SqliteControler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ namespace osu_StreamCompanion.Code.Core
public class SqliteControler : IDisposable, IMapDataStorer, CollectionManager.Interfaces.IMapDataManager
{
private readonly SqliteConnector _sqlConnector;
private HashSet<string> _md5List;
private Dictionary<string,int> _md5List;
public SqliteControler()
{
_sqlConnector = new SqliteConnector();
Expand Down Expand Up @@ -45,12 +45,12 @@ public void StartMassStoring()
{
lock (_sqlConnector)
{
string sql = "SELECT Md5 FROM (SELECT Md5 FROM `withID` UNION SELECT Md5 FROM `withoutID`)";
string sql = "SELECT Md5, MapId FROM (SELECT Md5, MapId FROM `withID` UNION SELECT Md5, MapId FROM `withoutID`)";
var reader = _sqlConnector.Query(sql);
_md5List = new HashSet<string>();
_md5List = new Dictionary<string, int>();
while (reader.Read())
{
_md5List.Add(reader.GetString(0));
_md5List.Add(reader.GetString(0), reader.GetInt32(1));
}
reader.Dispose();
_sqlConnector.StartMassStoring();
Expand Down Expand Up @@ -78,8 +78,10 @@ public void StoreBeatmap(Beatmap beatmap)
{
if (_sqlConnector.MassInsertIsActive)
{
if (_md5List.Contains(beatmap.Md5))
return;//no need to save same data.
var hash = beatmap.Md5;
if (_md5List.ContainsKey(hash))
if(_md5List[hash] == beatmap.MapId )
return;//no need to save same data.
}
_sqlConnector.StoreBeatmap(beatmap);
}
Expand Down

0 comments on commit 4e8903c

Please sign in to comment.