Skip to content

Commit

Permalink
Adding import/export story
Browse files Browse the repository at this point in the history
  • Loading branch information
@ayende
ayende committed May 14, 2010
1 parent f589571 commit 08b7ad9
Show file tree
Hide file tree
Showing 3 changed files with 304 additions and 0 deletions.
208 changes: 208 additions & 0 deletions Raven.Smuggler/Program.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,208 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Text;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;

namespace Raven.Smuggler
{
class Program
{
static void Main(string[] args)
{
if (args.Length != 3 || args[0] != "in" && args[0] != "out")
{
Console.WriteLine(@"
Raven Smuggler - Import/Export utility
Usage:
- Import the dump.raven file to a local instance:
Raven.Smuggler in http://localhost:8080/ dump.raven
- Export a local instance to dump.raven:
Raven.Smuggle out http://localhost:8080/ dump.raven
");
}

try
{
var instanceUrl = args[1];
if (instanceUrl.EndsWith("/") == false)
instanceUrl += "/";
var file = args[2];
switch (args[0])
{
case "in":
ImportData(instanceUrl, file);
break;
case "out":
ExportData(instanceUrl, file);
break;
}
}
catch (Exception e)
{
Console.WriteLine(e);
Environment.Exit(-1);
}
}

private static void ExportData(string instanceUrl, string file)
{
using (var streamWriter = new StreamWriter(new GZipStream(File.Create(file), CompressionMode.Compress)))
{
var jsonWriter = new JsonTextWriter(streamWriter)
{
Formatting = Formatting.Indented
};
jsonWriter.WriteStartObject();
jsonWriter.WritePropertyName("Indexes");
jsonWriter.WriteStartArray();
using (var webClient = new WebClient())
{
int totalCount = 0;
while (true)
{
var documents = webClient.DownloadString(instanceUrl + "indexes?pageSize=128&start=" + totalCount);
var array = JArray.Parse(documents);
if (array.Count == 0)
{
Console.WriteLine("Done with reading indexes, total: {0}", totalCount);
break;
}
totalCount += array.Count;
Console.WriteLine("Reading batch of {0,3} indexes, read so far: {1,10:#,#}", array.Count,
totalCount);
foreach (JToken item in array)
{
item.WriteTo(jsonWriter);
}
}
}
jsonWriter.WriteEndArray();
jsonWriter.WritePropertyName("Docs");
jsonWriter.WriteStartArray();
using (var webClient = new WebClient())
{
var lastEtag = Guid.Empty;
int totalCount = 0;
while (true)
{
var documents = webClient.DownloadString(instanceUrl + "docs?pageSize=128&etag=" + lastEtag);
var array = JArray.Parse(documents);
if (array.Count == 0)
{
Console.WriteLine("Done with reading documents, total: {0}", totalCount);
break;
}
totalCount += array.Count;
Console.WriteLine("Reading batch of {0,3} documents, read so far: {1,10:#,#}", array.Count,
totalCount);
foreach (JToken item in array)
{
item.WriteTo(jsonWriter);
}
lastEtag = Guid.Parse(array.Last.Value<JObject>("@metadata").Value<string>("@etag"));
}
}
jsonWriter.WriteEndArray();
jsonWriter.WriteEndObject();
streamWriter.Flush();
}
}

private static void ImportData(string instanceUrl, string file)
{
using (var streamReader = new StreamReader(new GZipStream(File.OpenRead(file), CompressionMode.Decompress)))
{
var jsonReader = new JsonTextReader(streamReader);
if (jsonReader.Read() == false)
return;
if (jsonReader.TokenType != JsonToken.StartObject)
throw new InvalidDataException("StartObject was expected");
// should read indexes now
if (jsonReader.Read() == false)
return;
if (jsonReader.TokenType != JsonToken.PropertyName)
throw new InvalidDataException("PropertyName was expected");
if (Equals("Indexes", jsonReader.Value) == false)
throw new InvalidDataException("Indexes property was expected");
if (jsonReader.Read() == false)
return;
if (jsonReader.TokenType != JsonToken.StartArray)
throw new InvalidDataException("StartArray was expected");
using (var webClient = new WebClient())
{
webClient.UseDefaultCredentials = true;
webClient.Credentials = CredentialCache.DefaultNetworkCredentials;
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var index = JToken.ReadFrom(jsonReader);
var indexName = index.Value<string>("name");
if(indexName.StartsWith("Raven/"))
continue;
using (var streamWriter = new StreamWriter(webClient.OpenWrite(instanceUrl + "indexes/" + indexName, "PUT")))
using (var jsonTextWriter = new JsonTextWriter(streamWriter))
{
index.Value<JObject>("definition").WriteTo(jsonTextWriter);
jsonTextWriter.Flush();
streamWriter.Flush();
}
}
}
// should read documents now
if (jsonReader.Read() == false)
return;
if (jsonReader.TokenType != JsonToken.PropertyName)
throw new InvalidDataException("PropertyName was expected");
if (Equals("Docs", jsonReader.Value) == false)
throw new InvalidDataException("Docs property was expected");
if (jsonReader.Read() == false)
return;
if (jsonReader.TokenType != JsonToken.StartArray)
throw new InvalidDataException("StartArray was expected");
var batch = new List<JObject>();
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
var document = JToken.ReadFrom(jsonReader);
batch.Add((JObject)document);
if (batch.Count > 128)
FlushBatch(instanceUrl, batch);
}
FlushBatch(instanceUrl, batch);
}
}

private static void FlushBatch(string instanceUrl, List<JObject> batch)
{
using (var webClient = new WebClient())
{
webClient.UseDefaultCredentials = true;
webClient.Credentials = CredentialCache.DefaultNetworkCredentials;
using (var stream = webClient.OpenWrite(instanceUrl + "bulk_docs", "POST"))
using (var streamWriter = new StreamWriter(stream))
using (var jsonTextWriter = new JsonTextWriter(streamWriter))
{
var commands = new JArray();
foreach (var doc in batch)
{
var metadata = doc.Value<JObject>("@metadata");
doc.Remove("@metadata");
commands.Add(new JObject(
new JProperty("Method", "PUT"),
new JProperty("Document", doc),
new JProperty("Metadata", metadata),
new JProperty("Key", metadata.Value<string>("@id"))
));
}
commands.WriteTo(jsonTextWriter);
jsonTextWriter.Flush();
streamWriter.Flush();
}
}
batch.Clear();
}
}
}
36 changes: 36 additions & 0 deletions Raven.Smuggler/Properties/AssemblyInfo.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;

// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("Raven.Smuggler")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Microsoft")]
[assembly: AssemblyProduct("Raven.Smuggler")]
[assembly: AssemblyCopyright("Copyright © Microsoft 2010")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]

// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]

// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("a8d9139b-d43f-496d-b36c-11141871be90")]

// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]
60 changes: 60 additions & 0 deletions Raven.Smuggler/Raven.Smuggler.csproj
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">x86</Platform>
<ProductVersion>8.0.30703</ProductVersion>
<SchemaVersion>2.0</SchemaVersion>
<ProjectGuid>{3E6401AC-3E33-4B61-A460-49953654A207}</ProjectGuid>
<OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Raven.Smuggler</RootNamespace>
<AssemblyName>Raven.Smuggler</AssemblyName>
<TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
<TargetFrameworkProfile>Client</TargetFrameworkProfile>
<FileAlignment>512</FileAlignment>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x86' ">
<PlatformTarget>x86</PlatformTarget>
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x86' ">
<PlatformTarget>x86</PlatformTarget>
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="Newtonsoft.Json">
<HintPath>..\SharedLibs\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

0 comments on commit 08b7ad9

Please sign in to comment.