Merge remote-tracking branch 'gh/next'

src/common/ceph_context.cc

@@ -183,7 +183,7 @@ void CephContext::do_command(std::string command, std::string args, bufferlist *
       std::string var = args;
       size_t pos = var.find(' ');
       if (pos == string::npos) {
-	jf.dump_string("error", "set_config syntax is 'set_config <var> <value>'");
+	jf.dump_string("error", "syntax error: 'config set <var> <value>'");
       } else {
 	std::string val = var.substr(pos+1);
 	var.resize(pos);
@@ -253,7 +253,7 @@ CephContext::CephContext(uint32_t module_type_)
   _admin_socket->register_command("2", _admin_hook, "");
   _admin_socket->register_command("perf schema", _admin_hook, "dump perfcounters schema");
   _admin_socket->register_command("config show", _admin_hook, "dump current config settings");
-  _admin_socket->register_command("config set", _admin_hook, "set_config <field> <val>: set a config settings");
+  _admin_socket->register_command("config set", _admin_hook, "config set <field> <val>: set a config variable");
   _admin_socket->register_command("log flush", _admin_hook, "flush log entries to log file");
   _admin_socket->register_command("log dump", _admin_hook, "dump recent log entries to log file");
   _admin_socket->register_command("log reopen", _admin_hook, "reopen log file");

src/mds/MDS.cc

@@ -780,10 +780,10 @@ void MDS::handle_command(MMonCommand *m)
 	  CDir *dir = in->get_dirfrag(frag_t());
 	  if (dir && dir->is_auth()) {
 	    mdcache->migrator->export_dir(dir, target);
-	  } else dout(0) << "bad migrate_dir path dirfrag frag_t() or dir not auth" << dendl;
-	} else dout(0) << "bad migrate_dir path" << dendl;
-      } else dout(0) << "bad migrate_dir target syntax" << dendl;
-    } else dout(0) << "bad migrate_dir syntax" << dendl;
+	  } else dout(0) << "bad export_dir path dirfrag frag_t() or dir not auth" << dendl;
+	} else dout(0) << "bad export_dir path" << dendl;
+      } else dout(0) << "bad export_dir target syntax" << dendl;
+    } else dout(0) << "bad export_dir syntax" << dendl;
   } 
   else if (m->cmd[0] == "cpu_profiler") {
     ostringstream ss;

src/mon/OSDMonitor.cc

@@ -3041,7 +3041,7 @@ bool OSDMonitor::prepare_command(MMonCommand *m)
 	      paxos->wait_for_commit(new Monitor::C_Command(mon, m, 0, rs, paxos->get_version()));
 	      return true;
 	    } else if (m->cmd[4] == "pg_num") {
-	      if (m->cmd.size() < 6 ||
+	      if (m->cmd.size() < 7 ||
 		  m->cmd[6] != "--allow-experimental-feature") {
 		ss << "increasing pg_num is currently experimental, add "
 		   << "--allow-experimental-feature as the last argument "

src/osd/OSD.cc

@@ -1000,7 +1000,7 @@ int OSD::init()
   assert(r == 0);
   test_ops_hook = new TestOpsSocketHook(&(this->service), this->store);
   r = admin_socket->register_command("setomapval", test_ops_hook,
-                              "setomap <pool-id> <obj-name> <key> <val>");
+                              "setomapval <pool-id> <obj-name> <key> <val>");
   assert(r == 0);
   r = admin_socket->register_command("rmomapkey", test_ops_hook,
                                "rmomapkey <pool-id> <obj-name> <key>");
@@ -1032,6 +1032,11 @@ int OSD::init()
 
   osd_lock.Lock();
 
+  dout(10) << "ensuring pgs have consumed prior maps" << dendl;
+  consume_map();
+  peering_wq.drain();
+
+  dout(10) << "done with init, starting boot process" << dendl;
   state = STATE_BOOTING;
   start_boot();
 

src/osd/PG.cc

@@ -5015,6 +5015,7 @@ void PG::handle_activate_map(RecoveryCtx *rctx)
   dout(10) << "handle_activate_map " << dendl;
   ActMap evt;
   recovery_state.handle_event(evt, rctx);
+  dirty_info = true;
 }
 
 void PG::handle_loaded(RecoveryCtx *rctx)

src/osd/ReplicatedPG.cc

@@ -3490,6 +3490,11 @@ void ReplicatedPG::do_osd_op_effects(OpContext *ctx)
   }
 }
 
+bool ReplicatedPG::have_temp_coll()
+{
+  return temp_created || osd->store->collection_exists(temp_coll);
+}
+
 coll_t ReplicatedPG::get_temp_coll(ObjectStore::Transaction *t)
 {
   if (temp_created)

src/osd/ReplicatedPG.h

@@ -1011,9 +1011,7 @@ class ReplicatedPG : public PG {
   coll_t temp_coll;
   coll_t get_temp_coll(ObjectStore::Transaction *t);
 public:
-  bool have_temp_coll() {
-    return temp_created;
-  }
+  bool have_temp_coll();
   coll_t get_temp_coll() {
     return temp_coll;
   }

src/rgw/rgw_admin.cc

@@ -41,6 +41,7 @@ void _usage()
   cerr << "  user rm                    remove user\n";
   cerr << "  user suspend               suspend a user\n";
   cerr << "  user enable                reenable user after suspension\n";
+  cerr << "  user check                 check user info\n";
   cerr << "  caps add                   add user capabilities\n";
   cerr << "  caps rm                    remove user capabilities\n";
   cerr << "  subuser create             create a new subuser\n" ;
@@ -55,6 +56,7 @@ void _usage()
   cerr << "  bucket rm                  remove bucket\n";
   cerr << "  bucket check               check bucket index\n";
   cerr << "  object rm                  remove object\n";
+  cerr << "  object unlink              unlink object from bucket index\n";
   cerr << "  cluster info               show cluster params info\n";
   cerr << "  pool add                   add an existing pool for data placement\n";
   cerr << "  pool rm                    remove an existing pool from data placement set\n";
@@ -133,6 +135,7 @@ enum {
   OPT_USER_RM,
   OPT_USER_SUSPEND,
   OPT_USER_ENABLE,
+  OPT_USER_CHECK,
   OPT_SUBUSER_CREATE,
   OPT_SUBUSER_MODIFY,
   OPT_SUBUSER_RM,
@@ -142,8 +145,8 @@ enum {
   OPT_BUCKET_LINK,
   OPT_BUCKET_UNLINK,
   OPT_BUCKET_STATS,
-  OPT_BUCKET_RM,
   OPT_BUCKET_CHECK,
+  OPT_BUCKET_RM,
   OPT_POLICY,
   OPT_POOL_ADD,
   OPT_POOL_RM,
@@ -155,6 +158,7 @@ enum {
   OPT_USAGE_TRIM,
   OPT_TEMP_REMOVE,
   OPT_OBJECT_RM,
+  OPT_OBJECT_UNLINK,
   OPT_GC_LIST,
   OPT_GC_PROCESS,
   OPT_CLUSTER_INFO,
@@ -227,6 +231,7 @@ static int get_cmd(const char *cmd, const char *prev_cmd, bool *need_more)
       strcmp(cmd, "key") == 0 ||
       strcmp(cmd, "buckets") == 0 ||
       strcmp(cmd, "bucket") == 0 ||
+      strcmp(cmd, "object") == 0 ||
       strcmp(cmd, "pool") == 0 ||
       strcmp(cmd, "pools") == 0 ||
       strcmp(cmd, "log") == 0 ||
@@ -259,6 +264,8 @@ static int get_cmd(const char *cmd, const char *prev_cmd, bool *need_more)
       return OPT_USER_SUSPEND;
     if (strcmp(cmd, "enable") == 0)
       return OPT_USER_ENABLE;
+    if (strcmp(cmd, "check") == 0)
+      return OPT_USER_CHECK;
   } else if (strcmp(prev_cmd, "subuser") == 0) {
     if (strcmp(cmd, "create") == 0)
       return OPT_SUBUSER_CREATE;
@@ -318,6 +325,8 @@ static int get_cmd(const char *cmd, const char *prev_cmd, bool *need_more)
   } else if (strcmp(prev_cmd, "object") == 0) {
     if (strcmp(cmd, "rm") == 0)
       return OPT_OBJECT_RM;
+    if (strcmp(cmd, "unlink") == 0)
+      return OPT_OBJECT_UNLINK;
   } else if (strcmp(prev_cmd, "cluster") == 0) {
     if (strcmp(cmd, "info") == 0)
       return OPT_CLUSTER_INFO;
@@ -574,6 +583,128 @@ enum ObjectKeyType {
   KEY_TYPE_S3,
 };
 
+static void check_bad_index_multipart(RGWRados *store, rgw_bucket& bucket, bool fix)
+{
+  int max = 1000;
+  string prefix;
+  string marker;
+  string delim;
+
+  map<string, bool> common_prefixes;
+  string ns = "multipart";
+
+  bool is_truncated;
+  list<string> objs_to_unlink;
+  map<string, bool> meta_objs;
+  map<string, string> all_objs;
+
+  do {
+    vector<RGWObjEnt> result;
+    int r = store->list_objects(bucket, max, prefix, delim, marker,
+				result, common_prefixes, false, ns,
+				&is_truncated, NULL);
+
+    if (r < 0) {
+      cerr << "failed to list objects in bucket=" << bucket << " err=" << cpp_strerror(-r) << std::endl;
+      return;
+    }
+
+    vector<RGWObjEnt>::iterator iter;
+    for (iter = result.begin(); iter != result.end(); ++iter) {
+      RGWObjEnt& ent = *iter;
+
+      rgw_obj obj(bucket, ent.name);
+      obj.set_ns(ns);
+
+      string& oid = obj.object;
+      marker = oid;
+
+      int pos = oid.find_last_of('.');
+      if (pos < 0)
+	continue;
+
+      string name = oid.substr(0, pos);
+      string suffix = oid.substr(pos + 1);
+
+      if (suffix.compare("meta") == 0) {
+	meta_objs[name] = true;
+      } else {
+	all_objs[oid] = name;
+      }
+    }
+
+  } while (is_truncated);
+
+  map<string, string>::iterator aiter;
+  for (aiter = all_objs.begin(); aiter != all_objs.end(); ++aiter) {
+    string& name = aiter->second;
+
+    if (meta_objs.find(name) == meta_objs.end()) {
+      objs_to_unlink.push_back(aiter->first);
+    }
+  }
+
+  if (objs_to_unlink.empty())
+    return;
+
+  if (!fix) {
+    cout << "Need to unlink the following objects from bucket=" << bucket << std::endl;
+  } else {
+    cout << "Unlinking the following objects from bucket=" << bucket << std::endl;
+  }
+  for (list<string>::iterator oiter = objs_to_unlink.begin(); oiter != objs_to_unlink.end(); ++oiter) {
+    cout << *oiter << std::endl;
+  }
+
+  if (fix) {
+    int r = store->remove_objs_from_index(bucket, objs_to_unlink);
+    if (r < 0) {
+      cerr << "ERROR: remove_obj_from_index() returned error: " << cpp_strerror(-r) << std::endl;
+    }
+  }
+}
+
+static void check_bad_user_bucket_mapping(RGWRados *store, const string& user_id, bool fix)
+{
+  RGWUserBuckets user_buckets;
+  int ret = rgw_read_user_buckets(store, user_id, user_buckets, false);
+  if (ret < 0) {
+    cerr << "failed to read user buckets: " << cpp_strerror(-ret) << std::endl;
+    return;
+  }
+
+  map<string, RGWBucketEnt>& buckets = user_buckets.get_buckets();
+  for (map<string, RGWBucketEnt>::iterator i = buckets.begin();
+       i != buckets.end();
+       ++i) {
+    RGWBucketEnt& bucket_ent = i->second;
+    rgw_bucket& bucket = bucket_ent.bucket;
+
+    RGWBucketInfo bucket_info;
+    int r = store->get_bucket_info(NULL, bucket.name, bucket_info);
+    if (r < 0) {
+      cerr << "could not get bucket info for bucket=" << bucket << std::endl;
+      continue;
+    }
+
+    rgw_bucket& actual_bucket = bucket_info.bucket;
+
+    if (actual_bucket.name.compare(bucket.name) != 0 ||
+        actual_bucket.pool.compare(bucket.pool) != 0 ||
+        actual_bucket.marker.compare(bucket.marker) != 0 ||
+        actual_bucket.bucket_id.compare(bucket.bucket_id) != 0) {
+      cout << "bucket info mismatch: expected " << actual_bucket << " got " << bucket << std::endl;
+      if (fix) {
+	cout << "fixing" << std::endl;
+	r = rgw_add_bucket(store, user_id, actual_bucket);
+	if (r < 0) {
+	  cerr << "failed to fix bucket: " << cpp_strerror(-r) << std::endl;
+	}
+      }
+    }
+  }
+}
+
 static int remove_object(RGWRados *store, rgw_bucket& bucket, std::string& object)
 {
   int ret = -EINVAL;
@@ -1627,7 +1758,19 @@ int main(int argc, char **argv)
     }
   }
 
+  if (opt_cmd == OPT_OBJECT_UNLINK) {
+    list<string> oid_list;
+    oid_list.push_back(object);
+    int ret = store->remove_objs_from_index(bucket, oid_list);
+    if (ret < 0) {
+      cerr << "ERROR: remove_obj_from_index() returned error: " << cpp_strerror(-ret) << std::endl;
+      return 1;
+    }
+  }
+
   if (opt_cmd == OPT_BUCKET_CHECK) {
+    check_bad_index_multipart(store, bucket, fix);
+
     map<RGWObjCategory, RGWBucketStats> existing_stats;
     map<RGWObjCategory, RGWBucketStats> calculated_stats;
 
@@ -1757,5 +1900,10 @@ int main(int argc, char **argv)
     store->params.dump(formatter);
     formatter->flush(cout);
   }
+
+  if (opt_cmd == OPT_USER_CHECK) {
+    check_bad_user_bucket_mapping(store, user_id, fix);
+  }
+
   return 0;
 }

src/rgw/rgw_op.cc

@@ -836,6 +836,27 @@ void RGWCreateBucket::execute()
 
   existed = (ret == -EEXIST);
 
+  if (existed) {
+    /* bucket already existed, might have raced with another bucket creation, or
+     * might be partial bucket creation that never completed. Read existing bucket
+     * info, verify that the reported bucket owner is the current user.
+     * If all is ok then update the user's list of buckets
+     */
+    RGWBucketInfo info;
+    map<string, bufferlist> attrs;
+    int r = store->get_bucket_info(NULL, s->bucket.name, info, &attrs);
+    if (r < 0) {
+      ldout(s->cct, 0) << "ERROR: get_bucket_info on bucket=" << s->bucket.name << " returned err=" << r << " after create_bucket returned -EEXIST" << dendl;
+      ret = r;
+      return;
+    }
+    if (info.owner.compare(s->user.user_id) != 0) {
+      ret = -ERR_BUCKET_EXISTS;
+      return;
+    }
+    s->bucket = info.bucket;
+  }
+
   ret = rgw_add_bucket(store, s->user.user_id, s->bucket);
   if (ret && !existed && ret != -EEXIST)   /* if it exists (or previously existed), don't remove it! */
     rgw_remove_user_bucket_info(store, s->user.user_id, s->bucket);

src/rgw/rgw_rados.cc

@@ -806,6 +806,9 @@ int RGWRados::create_bucket(string& owner, rgw_bucket& bucket,
   info.bucket = bucket;
   info.owner = owner;
   ret = store_bucket_info(info, &attrs, exclusive);
+  if (ret == -EEXIST) {
+    io_ctx.remove(dir_oid);
+  }
 
   return ret;
 }
@@ -3634,6 +3637,38 @@ int RGWRados::cls_obj_usage_log_trim(string& oid, string& user, uint64_t start_e
   return r;
 }
 
+int RGWRados::remove_objs_from_index(rgw_bucket& bucket, list<string>& oid_list)
+{
+  librados::IoCtx io_ctx;
+
+  int r = open_bucket_ctx(bucket, io_ctx);
+  if (r < 0)
+    return r;
+
+  string dir_oid = dir_oid_prefix;
+  dir_oid.append(bucket.marker);
+
+  bufferlist updates;
+
+  list<string>::iterator iter;
+
+  for (iter = oid_list.begin(); iter != oid_list.end(); ++iter) {
+    string& oid = *iter;
+    dout(2) << "RGWRados::remove_objs_from_index bucket=" << bucket << " oid=" << oid << dendl;
+    rgw_bucket_dir_entry entry;
+    entry.epoch = (uint64_t)-1; // ULLONG_MAX, needed to that objclass doesn't skip out request
+    entry.name = oid;
+    updates.append(CEPH_RGW_REMOVE);
+    ::encode(entry, updates);
+  }
+
+  bufferlist out;
+
+  r = io_ctx.exec(dir_oid, "rgw", "dir_suggest_changes", updates, out);
+
+  return r;
+}
+
 int RGWRados::check_disk_state(librados::IoCtx io_ctx,
                                rgw_bucket& bucket,
                                rgw_bucket_dir_entry& list_state,